Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

88.6 KB
Raw Blame
mod emit;
mod gate;
#[cfg(test)]
mod tests;

use moli_cookie_jar::{StoredCookieQueryReport, StoredCookieSetReport};
use moli_core::RendererOutputFence;
use moli_core::page::{
    NavigationRedirect, RendererMainDocumentCommit, RendererPageCreationArtifacts,
    RendererPendingDownloadActivation, RendererRuntimeRealmInfo, SubresourceRequestInitiatorType,
};
use moli_fetch::{
    NegotiatedHttpVersion, NetworkExchangeObservation, NetworkObservationJournal, RedirectInfo,
    RequestHeaders, ResponseHead, StreamingRawResponse,
};
use std::sync::Arc;
use url::Url;

use crate::automation::DevToolsRequestFailure;
use crate::conn::{
    BackgroundEventSender, BackgroundProtocolEvent, CapturedBody, CdpConnection,
    CommittedNavigationPage, CompletedDownloadBodyArtifact, DownloadNavigation, LoadedNavigation,
    NavigationDispatchState, NavigationLoadOutcome, NavigationRequestBlocked, NoContentNavigation,
    ResponseCommitReady, TargetRuntimeSlot,
};

#[cfg(test)]
use gate::MainDocumentProgressDrain;
pub(crate) use gate::{MainDocumentProgressBackgroundEventBarrier, MainDocumentProgressGate};
use gate::{
    MainDocumentProgressEmission, MainDocumentProgressEventBatch,
    MainDocumentProgressOutputBoundary, MainDocumentProgressOutputTarget,
    MainDocumentProgressPhase, MainDocumentProgressQueueHandle, MainDocumentProgressSource,
};

pub(crate) struct MaterializedLoadedDocumentProgress {
    pub(crate) page: CommittedNavigationPage,
    pub(crate) pending_download: Option<RendererPendingDownloadActivation>,
    pub(crate) page_creation_artifacts: RendererPageCreationArtifacts,
    pub(crate) final_url: Url,
    pub(crate) response_headers: Vec<(String, Vec<u8>)>,
    pub(crate) response_from_cache: bool,
    pub(crate) main_document_body: Option<CapturedBody>,
    pub(crate) initial_runtime_realms: Vec<RendererRuntimeRealmInfo>,
    pub(crate) renderer_output_predecessor: Option<RendererOutputFence>,
    pub(crate) main_document_commit: Option<Arc<RendererMainDocumentCommit>>,
    pub(crate) progress_gate: MainDocumentProgressGate,
    pub(crate) network_error_page: Option<crate::conn::NetworkErrorPageNavigation>,
}

pub(crate) struct MaterializedDownloadDocumentProgress {
    pub(crate) final_url: Url,
    pub(crate) progress_gate: MainDocumentProgressGate,
    pub(crate) body_artifact: CompletedDownloadBodyArtifact,
}

pub(crate) struct MaterializedFailedDocumentProgress {
    pub(crate) error_text: String,
    pub(crate) document_policy: FailedNavigationDocumentPolicy,
    pub(crate) response_mode: FailedNavigationResponseMode,
    pub(crate) progress_gate: MainDocumentProgressGate,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum FailedNavigationResponseMode {
    ProtocolError,
    CdpErrorTextResult,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum FailedNavigationDocumentPolicy {
    InvalidateCommittedDocument,
    PreserveCommittedDocument,
}

impl FailedNavigationDocumentPolicy {
    fn for_navigation_error(error: &anyhow::Error) -> Self {
        // Decide from the cause before projecting diagnostic text. Context and
        // shared body-capture wrappers must not turn cancellation into failure.
        if moli_fetch::is_fetch_cancelled(error)
            || matches!(
                error.downcast_ref::<DevToolsRequestFailure>(),
                Some(DevToolsRequestFailure::Aborted)
            )
            || error.chain().any(|cause| {
                matches!(
                    cause.downcast_ref::<DevToolsRequestFailure>(),
                    Some(DevToolsRequestFailure::Aborted)
                )
            })
        {
            Self::PreserveCommittedDocument
        } else {
            Self::InvalidateCommittedDocument
        }
    }

    pub(crate) fn invalidates_committed_document(self) -> bool {
        matches!(self, Self::InvalidateCommittedDocument)
    }
}

pub(crate) enum MaterializedNavigationLoadOutcome {
    ResponseCommitReady(Box<ResponseCommitReady>),
    Loaded(Box<MaterializedLoadedDocumentProgress>),
    Download(MaterializedDownloadDocumentProgress),
    Failed(MaterializedFailedDocumentProgress),
}

#[cfg(test)]
pub(crate) fn empty_main_document_progress_gate_for_test() -> MainDocumentProgressGate {
    MainDocumentProgressGate::from_queue(MainDocumentProgressQueueHandle::new(
        MainDocumentProgressDrain::new(),
    ))
}

#[derive(Debug)]
pub(crate) struct CompletedDocumentProgressTransfer {
    body: CompletedDocumentProgressBody,
    network_progress: MainDocumentBodyNetworkProgress,
}

#[derive(Debug)]
enum CompletedDocumentProgressBody {
    Captured { body: CapturedBody, synthetic: bool },
    Pending,
}

impl CompletedDocumentProgressTransfer {
    pub(crate) fn new_captured(
        body: CapturedBody,
        synthetic: bool,
        network_progress: MainDocumentBodyNetworkProgress,
    ) -> Self {
        Self {
            body: CompletedDocumentProgressBody::Captured { body, synthetic },
            network_progress,
        }
    }

    pub(crate) fn new_pending_body(network_progress: MainDocumentBodyNetworkProgress) -> Self {
        Self {
            body: CompletedDocumentProgressBody::Pending,
            network_progress,
        }
    }

    fn captured_body(&self) -> Option<CapturedBody> {
        match &self.body {
            CompletedDocumentProgressBody::Captured { body, .. } => Some(body.clone()),
            CompletedDocumentProgressBody::Pending => None,
        }
    }

    fn into_progress_gate(
        self,
        conn: &mut CdpConnection,
        state: &NavigationDispatchState,
        final_url: &Url,
    ) -> MainDocumentProgressGate {
        let queue = completed_document_body_progress_queue(self, conn, state, final_url);
        MainDocumentProgressGate::from_queue(queue)
    }

    fn into_parts(
        self,
    ) -> (
        CompletedDocumentProgressBody,
        MainDocumentBodyNetworkProgress,
    ) {
        (self.body, self.network_progress)
    }

    #[cfg(test)]
    pub(crate) fn body(&self) -> &CapturedBody {
        match &self.body {
            CompletedDocumentProgressBody::Captured { body, .. } => body,
            CompletedDocumentProgressBody::Pending => {
                panic!("test navigation body is still pending")
            }
        }
    }

    #[cfg(test)]
    pub(crate) fn completed_body_network_events(
        &self,
    ) -> Option<&CompletedMainDocumentNetworkEvents> {
        self.network_progress.as_completed_body()
    }
}

#[derive(Debug)]
pub(crate) struct CompletedDownloadProgressTransfer {
    body: CompletedDownloadProgressBody,
    network_events: CompletedMainDocumentNetworkEvents,
}

#[derive(Debug)]
enum CompletedDownloadProgressBody {
    Buffered(Vec<u8>),
    Streaming(Box<StreamingRawResponse>),
}

impl CompletedDownloadProgressTransfer {
    pub(crate) fn new(body: Vec<u8>, network_events: CompletedMainDocumentNetworkEvents) -> Self {
        Self {
            body: CompletedDownloadProgressBody::Buffered(body),
            network_events,
        }
    }

    pub(crate) fn new_streaming(
        response: StreamingRawResponse,
        network_events: CompletedMainDocumentNetworkEvents,
    ) -> Self {
        Self {
            body: CompletedDownloadProgressBody::Streaming(Box::new(response)),
            network_events,
        }
    }

    fn into_progress_gate_and_artifact(
        self,
        conn: &CdpConnection,
        state: &NavigationDispatchState,
        final_url: &Url,
    ) -> (MainDocumentProgressGate, CompletedDownloadBodyArtifact) {
        let response_headers = self.network_events.response_headers.clone();
        let progress = CompletedDownloadBodyNetworkProgress {
            encoded_data_length: completed_download_body_len_hint(&self.body, &response_headers),
            network_events: self.network_events,
        };
        let queue = completed_download_body_progress_queue(progress, conn, state, final_url);
        let body = match self.body {
            CompletedDownloadProgressBody::Buffered(body) => {
                crate::conn::CompletedDownloadBody::Buffered(body)
            }
            CompletedDownloadProgressBody::Streaming(response) => {
                crate::conn::CompletedDownloadBody::Streaming(response)
            }
        };
        (
            MainDocumentProgressGate::from_queue(queue),
            CompletedDownloadBodyArtifact::from_body(body, response_headers),
        )
    }
}

fn completed_download_body_len_hint(
    body: &CompletedDownloadProgressBody,
    response_headers: &[(String, Vec<u8>)],
) -> usize {
    match body {
        CompletedDownloadProgressBody::Buffered(body) => body.len(),
        CompletedDownloadProgressBody::Streaming(_) => response_headers
            .iter()
            .find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
            .and_then(|(_, value)| moli_fetch::decode_header_value(value).trim().parse().ok())
            .unwrap_or_default(),
    }
}

#[derive(Debug)]
struct CompletedDownloadBodyNetworkProgress {
    encoded_data_length: usize,
    network_events: CompletedMainDocumentNetworkEvents,
}

impl CompletedDownloadBodyNetworkProgress {
    fn len(&self) -> usize {
        self.encoded_data_length
    }

    fn into_network_events(self) -> CompletedMainDocumentNetworkEvents {
        self.network_events
    }
}

pub(crate) fn failed_navigation_progress_gate(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    error_text: &str,
) -> MainDocumentProgressGate {
    MainDocumentProgressGate::from_queue(MainDocumentProgressQueueHandle::from_source(
        MainDocumentProgressSource::failed_navigation(observed_navigation_failure_event(
            conn, state, error_text,
        )),
    ))
}

fn network_error_page_progress_gate(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    error_text: &str,
) -> MainDocumentProgressGate {
    let failure_event = observed_navigation_failure_event(conn, state, error_text);
    let finish_event = observed_navigation_finished_event(conn, state);
    MainDocumentProgressGate::from_queue(MainDocumentProgressQueueHandle::from_source(
        MainDocumentProgressSource::error_page(failure_event, finish_event),
    ))
}

fn materialize_failed_navigation_progress(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    error_text: String,
    document_policy: FailedNavigationDocumentPolicy,
    response_mode: FailedNavigationResponseMode,
) -> MaterializedFailedDocumentProgress {
    let progress_gate = failed_navigation_progress_gate(conn, state, &error_text);
    MaterializedFailedDocumentProgress {
        error_text,
        document_policy,
        response_mode,
        progress_gate,
    }
}

pub(crate) fn materialize_loaded_navigation_progress(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    navigation: LoadedNavigation<impl Into<CommittedNavigationPage>>,
) -> MaterializedLoadedDocumentProgress {
    let LoadedNavigation {
        page,
        pending_download,
        page_creation_artifacts,
        final_url,
        response_headers,
        response_from_cache,
        initial_runtime_realms,
        renderer_output_predecessor,
        main_document_commit,
        document_progress_transfer,
        network_error_page,
        ..
    } = navigation;
    let main_document_body = document_progress_transfer.captured_body();
    let progress_gate = match network_error_page.as_ref() {
        Some(error_page) => network_error_page_progress_gate(conn, state, error_page.error_text()),
        None => document_progress_transfer.into_progress_gate(conn, state, &final_url),
    };
    MaterializedLoadedDocumentProgress {
        page: page.into(),
        pending_download,
        page_creation_artifacts,
        final_url,
        response_headers,
        response_from_cache,
        main_document_body,
        initial_runtime_realms,
        renderer_output_predecessor,
        main_document_commit,
        progress_gate,
        network_error_page,
    }
}

fn materialize_navigation_load_outcome(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    navigation: NavigationLoadOutcome,
) -> MaterializedNavigationLoadOutcome {
    match navigation {
        NavigationLoadOutcome::ResponseCommitReady(navigation) => {
            MaterializedNavigationLoadOutcome::ResponseCommitReady(navigation)
        }
        NavigationLoadOutcome::Loaded(navigation) => {
            MaterializedNavigationLoadOutcome::Loaded(Box::new(
                materialize_loaded_navigation_progress(conn, state, *navigation),
            ))
        }
        NavigationLoadOutcome::Download(navigation) => MaterializedNavigationLoadOutcome::Download(
            materialize_download_navigation_progress(conn, state, *navigation),
        ),
        NavigationLoadOutcome::NoContent(navigation) => {
            materialize_no_content_navigation_progress(conn, state, *navigation)
        }
        NavigationLoadOutcome::NetworkFailure(error_text) => {
            MaterializedNavigationLoadOutcome::Failed(materialize_failed_navigation_progress(
                conn,
                state,
                error_text,
                FailedNavigationDocumentPolicy::InvalidateCommittedDocument,
                FailedNavigationResponseMode::CdpErrorTextResult,
            ))
        }
    }
}

pub(crate) fn materialize_navigation_load_result(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    navigation: anyhow::Result<NavigationLoadOutcome>,
) -> MaterializedNavigationLoadOutcome {
    match navigation {
        Ok(navigation) => materialize_navigation_load_outcome(conn, state, navigation),
        Err(error) => {
            let document_policy = FailedNavigationDocumentPolicy::for_navigation_error(&error);
            tracing::debug!(
                error = ?error,
                session_id = state.owner.session_id(),
                "navigation load failed"
            );
            let error_text = match error.downcast_ref::<NavigationRequestBlocked>() {
                Some(blocked) => blocked.to_string(),
                None => format!("{error:#}"),
            };
            MaterializedNavigationLoadOutcome::Failed(materialize_failed_navigation_progress(
                conn,
                state,
                error_text,
                document_policy,
                FailedNavigationResponseMode::ProtocolError,
            ))
        }
    }
}

pub(crate) fn materialize_navigation_failure_preserving_committed_document(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    error_text: String,
) -> MaterializedNavigationLoadOutcome {
    MaterializedNavigationLoadOutcome::Failed(materialize_failed_navigation_progress(
        conn,
        state,
        error_text,
        FailedNavigationDocumentPolicy::PreserveCommittedDocument,
        FailedNavigationResponseMode::ProtocolError,
    ))
}

fn materialize_no_content_navigation_progress(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    navigation: NoContentNavigation,
) -> MaterializedNavigationLoadOutcome {
    let error_text = moli_fetch::NET_ERR_ABORTED_ERROR_TEXT;
    let context = CompletedMainDocumentProgressContext::for_navigation(
        conn,
        state,
        completed_body_main_document_network_request_id(
            main_document_network_observed(conn, state.session_id.as_deref()),
            state.request_id.clone(),
        ),
    );
    let batches = MainDocumentNavigationProgressEventBatches::new(
        context.request_and_redirect_progress_events(&navigation.network_events),
        context.response_received_progress_events(
            &navigation.network_events,
            &navigation.final_url,
            0,
        ),
        observed_navigation_failure_event(conn, state, error_text)
            .into_iter()
            .collect(),
    );
    let queue = MainDocumentProgressQueueHandle::from_source(
        MainDocumentProgressSource::completed_body(batches),
    );
    // No Document commit will advance visibility for this response.
    queue.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
    let progress_gate = MainDocumentProgressGate::from_queue(queue);
    MaterializedNavigationLoadOutcome::Failed(MaterializedFailedDocumentProgress {
        error_text: error_text.to_owned(),
        document_policy: FailedNavigationDocumentPolicy::PreserveCommittedDocument,
        response_mode: FailedNavigationResponseMode::CdpErrorTextResult,
        progress_gate,
    })
}

fn materialize_download_navigation_progress(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    navigation: DownloadNavigation,
) -> MaterializedDownloadDocumentProgress {
    let DownloadNavigation {
        final_url,
        progress_transfer,
    } = navigation;
    let (progress_gate, body_artifact) =
        progress_transfer.into_progress_gate_and_artifact(conn, state, &final_url);
    MaterializedDownloadDocumentProgress {
        final_url,
        progress_gate,
        body_artifact,
    }
}

fn completed_document_body_progress_queue(
    transfer: CompletedDocumentProgressTransfer,
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    final_url: &Url,
) -> MainDocumentProgressQueueHandle {
    let owner_session_id = state.session_id.as_deref();
    let session_ids = main_document_network_event_session_ids(conn, owner_session_id);
    let network_observed = main_document_network_observed(conn, owner_session_id);
    let (body, body_progress_source) = transfer.into_parts();
    let encoded_data_length = match body {
        CompletedDocumentProgressBody::Captured { body, synthetic } => {
            let encoded_data_length = body.len();
            let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
                owner_session_id,
                crate::automation::DevToolsNetworkDataType::Response,
                encoded_data_length,
            );
            let collection_was_gated = conn.network_data_collection_is_gated_for_body(
                crate::automation::DevToolsNetworkDataType::Response,
            );
            if network_observed
                && !synthetic
                && let Some(request_id) = state.request_id.clone()
            {
                conn.record_collected_network_data_body(
                    request_id,
                    crate::automation::DevToolsNetworkDataType::Response,
                    body.clone(),
                    collector_ids.iter().cloned(),
                    collection_was_gated,
                );
            }
            let _ = conn
                .runtime_session_owner_slot_mut(owner_session_id)
                .and_then(|runtime_slot| {
                    record_main_document_response_body_for_network(
                        runtime_slot,
                        network_observed,
                        state.request_id.clone(),
                        &session_ids,
                        collector_ids,
                        collection_was_gated,
                        synthetic,
                        &body,
                    )
                });
            encoded_data_length
        }
        CompletedDocumentProgressBody::Pending => {
            record_pending_main_document_response_body(conn, state, &session_ids);
            0
        }
    };
    let event_request_id =
        completed_body_main_document_network_request_id(network_observed, state.request_id.clone());
    completed_or_streaming_document_progress_queue(
        conn,
        state,
        network_observed,
        event_request_id,
        body_progress_source.into_completed_body_events(),
        final_url,
        encoded_data_length,
    )
}

fn completed_download_body_progress_queue(
    progress: CompletedDownloadBodyNetworkProgress,
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    final_url: &Url,
) -> MainDocumentProgressQueueHandle {
    let network_observed = main_document_network_observed(conn, state.session_id.as_deref());
    let encoded_data_length = progress.len();
    completed_or_streaming_document_progress_queue(
        conn,
        state,
        network_observed,
        state.request_id.clone(),
        Some(progress.into_network_events()),
        final_url,
        encoded_data_length,
    )
}

fn completed_or_streaming_document_progress_queue(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    network_enabled: bool,
    request_id: Option<String>,
    events: Option<CompletedMainDocumentNetworkEvents>,
    final_url: &Url,
    encoded_data_length: usize,
) -> MainDocumentProgressQueueHandle {
    let Some(events) = events else {
        return MainDocumentProgressQueueHandle::from_source(
            MainDocumentProgressSource::streaming(),
        );
    };
    let context = CompletedMainDocumentProgressContext::for_navigation(
        conn,
        state,
        completed_body_main_document_network_request_id(network_enabled, request_id),
    );
    MainDocumentProgressQueueHandle::from_source(MainDocumentProgressSource::completed_body(
        context.event_batches(&events, final_url, encoded_data_length),
    ))
}

#[derive(Clone)]
struct MainDocumentLiveNetworkProgressSource {
    sender: Option<BackgroundEventSender>,
    progress_queue: MainDocumentProgressQueueHandle,
    session_ids: Vec<Option<String>>,
    request_id: String,
    loader_id: String,
    frame_id: String,
    timestamp: f64,
    initial_request_headers: Vec<(String, String)>,
    initial_request_cookie_report: Option<StoredCookieQueryReport>,
}

#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum MainDocumentResponseVisibility {
    #[default]
    Immediate,
    AfterResponseStageContinue,
}

#[derive(Clone, Debug, Default)]
pub struct MainDocumentBodyProgressSource {
    live_source: Option<MainDocumentLiveNetworkProgressSource>,
    response_visibility: MainDocumentResponseVisibility,
}

impl MainDocumentBodyProgressSource {
    fn from_live_source(
        live_source: Option<MainDocumentLiveNetworkProgressSource>,
        response_visibility: MainDocumentResponseVisibility,
    ) -> Self {
        Self {
            live_source,
            response_visibility,
        }
    }

    pub(crate) fn body_network_progress_for_response_head(
        &self,
        request_method: &str,
        request_headers: &RequestHeaders,
        head: &ResponseHead,
        network_observation_journal: NetworkObservationJournal,
    ) -> MainDocumentBodyNetworkProgress {
        let network_extra_info_available = !network_observation_journal.is_empty();
        let request_headers = request_headers.to_byte_strings();
        self.emit_response_metadata(
            request_method,
            &request_headers,
            head.request_cookie_report.as_ref(),
            &head.redirect_chain,
            &head.final_url,
            head.status,
            &head.headers,
            &head.cookie_set_reports,
            &network_observation_journal,
            network_extra_info_available,
            head.from_cache,
            head.negotiated_http_version,
        );
        self.body_network_progress_for_completed_events(
            CompletedMainDocumentNetworkEvents::new(
                request_method.to_owned(),
                request_headers,
                head.request_cookie_report.clone(),
                head.status,
                head.headers.clone(),
                head.cookie_set_reports.clone(),
                head.redirect_chain
                    .iter()
                    .cloned()
                    .map(Into::into)
                    .collect(),
                network_extra_info_available,
                head.from_cache,
            )
            .with_negotiated_http_version(head.negotiated_http_version)
            .with_network_observation_journal(network_observation_journal),
        )
    }

    pub(crate) fn emit_response_metadata(
        &self,
        final_request_method: &str,
        final_request_headers: &[(String, String)],
        final_request_cookie_report: Option<&StoredCookieQueryReport>,
        redirect_chain: &[RedirectInfo],
        final_url: &Url,
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        response_cookie_reports: &[StoredCookieSetReport],
        network_observation_journal: &NetworkObservationJournal,
        network_extra_info_available: bool,
        response_from_cache: bool,
        negotiated_http_version: Option<NegotiatedHttpVersion>,
    ) {
        let Some(live_source) = self.live_source.as_ref() else {
            return;
        };
        match self.response_visibility {
            MainDocumentResponseVisibility::Immediate => {
                live_source.emit_redirect_requests(
                    final_request_method,
                    None,
                    final_request_headers,
                    final_request_cookie_report,
                    redirect_chain,
                    network_observation_journal,
                    network_extra_info_available,
                );
                live_source.emit_response_received(
                    final_url,
                    response_status,
                    response_headers,
                    response_cookie_reports,
                    network_observation_journal,
                    redirect_chain.len(),
                    network_extra_info_available,
                    response_from_cache,
                    negotiated_http_version,
                );
            }
            MainDocumentResponseVisibility::AfterResponseStageContinue => {
                live_source.emit_response_received_without_extra_info(
                    final_url,
                    response_status,
                    response_headers,
                    network_extra_info_available,
                    response_from_cache,
                    negotiated_http_version,
                );
            }
        }
    }

    pub(crate) fn emit_response_without_extra_info_into_background_events(
        &self,
        out: &mut Vec<BackgroundProtocolEvent>,
        final_url: &Url,
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        response_from_cache: bool,
    ) {
        let Some(live_source) = self.live_source.as_ref() else {
            return;
        };
        let mut output = MainDocumentProgressOutputTarget::background_events(out);
        live_source.send_progress_events_into_output(
            MainDocumentProgressPhase::ResponseReceived,
            vec![MainDocumentNavigationProgressEvent::ResponseReceived {
                target: live_source.progress_target(),
                final_url: final_url.clone(),
                status: response_status,
                headers: response_headers.to_vec(),
                cookie_set_reports: Vec::new(),
                extra_info_status: response_status,
                extra_info_headers: response_headers.to_vec(),
                network_extra_info_available: false,
                emit_extra_info: false,
                encoded_data_length: 0,
                from_cache: response_from_cache,
                negotiated_http_version: None,
                has_extra_info: false,
            }],
            &mut output,
        );
    }

    pub(crate) fn emit_failed_initial_request_extra_info(
        &self,
        network_observation_journal: &NetworkObservationJournal,
    ) {
        let Some(live_source) = self.live_source.as_ref() else {
            return;
        };
        let [exchange] = network_observation_journal.exchanges() else {
            return;
        };
        if exchange.response().is_some() {
            return;
        }
        let request = exchange.request();
        let cookie_report = request
            .cookie_report()
            .cloned()
            .or_else(|| live_source.initial_request_cookie_report.clone());
        live_source.send_progress_events(
            MainDocumentProgressPhase::RequestStarted,
            vec![request_extra_info_event(
                live_source.progress_target(),
                request.headers().to_vec(),
                cookie_report,
            )],
        );
    }

    pub(crate) fn emit_failed_request_progress(
        &self,
        final_request_method: &str,
        request_body: Option<&str>,
        final_request_headers: &[(String, String)],
        redirect_chain: &[RedirectInfo],
        network_observation_journal: &NetworkObservationJournal,
    ) {
        let Some(live_source) = self.live_source.as_ref() else {
            return;
        };
        let final_exchange = navigation_exchange_group(
            network_observation_journal,
            redirect_chain.len(),
            redirect_chain.len(),
        )
        .and_then(|group| group.last());
        let final_network_extra_info_available =
            final_exchange.is_some_and(|exchange| exchange.response().is_none());
        let final_request_cookie_report = final_exchange
            .and_then(|exchange| exchange.request().cookie_report())
            .cloned();
        let events = live_source.redirect_request_events(
            final_request_method,
            request_body,
            final_request_headers,
            final_request_cookie_report.as_ref(),
            redirect_chain,
            network_observation_journal,
            final_network_extra_info_available,
        );
        live_source.send_progress_events(MainDocumentProgressPhase::RequestStarted, events);
    }

    pub(crate) fn emit_response_extra_info_before_pause(
        &self,
        out: &mut Vec<BackgroundProtocolEvent>,
        final_request_method: &str,
        final_request_headers: &[(String, String)],
        final_request_cookie_report: Option<&StoredCookieQueryReport>,
        redirect_chain: &[RedirectInfo],
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        response_cookie_reports: &[StoredCookieSetReport],
        network_observation_journal: &NetworkObservationJournal,
        network_extra_info_available: bool,
    ) {
        let Some(live_source) = self.live_source.as_ref() else {
            return;
        };
        if self.response_visibility != MainDocumentResponseVisibility::AfterResponseStageContinue {
            return;
        }

        let mut output = MainDocumentProgressOutputTarget::background_events(out);
        live_source.emit_redirect_requests_into_output(
            &mut output,
            final_request_method,
            None,
            final_request_headers,
            final_request_cookie_report,
            redirect_chain,
            network_observation_journal,
            network_extra_info_available,
        );
        if network_extra_info_available {
            let (response_status, response_headers) = observed_response_metadata(
                network_observation_journal,
                redirect_chain.len(),
                redirect_chain.len(),
                response_status,
                response_headers,
            );
            live_source.emit_response_received_extra_info_into_output(
                &mut output,
                response_status,
                &response_headers,
                response_cookie_reports,
            );
        }
    }

    pub(crate) fn emit_body_finished(&self, encoded_data_length: usize) {
        if let Some(live_source) = self.live_source.as_ref() {
            live_source.emit_body_finished(encoded_data_length);
        }
    }

    pub(crate) fn body_network_progress_for_completed_events(
        &self,
        completed_events: CompletedMainDocumentNetworkEvents,
    ) -> MainDocumentBodyNetworkProgress {
        if self
            .live_source
            .as_ref()
            .is_some_and(|live_source| live_source.sender.is_some())
        {
            MainDocumentBodyNetworkProgress::StreamingBody
        } else {
            MainDocumentBodyNetworkProgress::CompletedBody(Box::new(
                self.completed_body_network_events(completed_events),
            ))
        }
    }
    pub(crate) fn completed_body_network_events(
        &self,
        mut events: CompletedMainDocumentNetworkEvents,
    ) -> CompletedMainDocumentNetworkEvents {
        events.response_stage_metadata_already_emitted =
            self.response_visibility == MainDocumentResponseVisibility::AfterResponseStageContinue;
        events
    }
}

#[derive(Debug, Clone)]
pub(crate) struct CompletedMainDocumentNetworkEvents {
    pub(crate) request_method: String,
    pub(crate) request_headers: Vec<(String, String)>,
    pub(crate) final_request_cookie_report: Option<StoredCookieQueryReport>,
    pub(crate) response_status: u16,
    pub(crate) response_headers: Vec<(String, Vec<u8>)>,
    pub(crate) response_cookie_reports: Vec<StoredCookieSetReport>,
    pub(crate) redirect_chain: Vec<NavigationRedirect>,
    pub(crate) network_extra_info_available: bool,
    pub(crate) response_from_cache: bool,
    negotiated_http_version: Option<NegotiatedHttpVersion>,
    network_observation_journal: NetworkObservationJournal,
    response_stage_metadata_already_emitted: bool,
}

impl CompletedMainDocumentNetworkEvents {
    pub(crate) fn new(
        request_method: String,
        request_headers: Vec<(String, String)>,
        final_request_cookie_report: Option<StoredCookieQueryReport>,
        response_status: u16,
        response_headers: Vec<(String, Vec<u8>)>,
        response_cookie_reports: Vec<StoredCookieSetReport>,
        redirect_chain: Vec<NavigationRedirect>,
        network_extra_info_available: bool,
        response_from_cache: bool,
    ) -> Self {
        Self {
            request_method,
            request_headers,
            final_request_cookie_report,
            response_status,
            response_headers,
            response_cookie_reports,
            redirect_chain,
            network_extra_info_available,
            response_from_cache,
            negotiated_http_version: None,
            network_observation_journal: NetworkObservationJournal::default(),
            response_stage_metadata_already_emitted: false,
        }
    }

    pub(crate) fn with_negotiated_http_version(
        mut self,
        negotiated_http_version: Option<NegotiatedHttpVersion>,
    ) -> Self {
        self.negotiated_http_version = negotiated_http_version;
        self
    }

    pub(crate) fn with_network_observation_journal(
        mut self,
        network_observation_journal: NetworkObservationJournal,
    ) -> Self {
        self.network_observation_journal = network_observation_journal;
        self
    }
}

#[derive(Debug)]
pub(crate) enum MainDocumentBodyNetworkProgress {
    CompletedBody(Box<CompletedMainDocumentNetworkEvents>),
    StreamingBody,
}

impl MainDocumentBodyNetworkProgress {
    pub(crate) fn into_completed_body_events(self) -> Option<CompletedMainDocumentNetworkEvents> {
        match self {
            Self::CompletedBody(events) => Some(*events),
            Self::StreamingBody => None,
        }
    }

    #[cfg(test)]
    pub(crate) fn as_completed_body(&self) -> Option<&CompletedMainDocumentNetworkEvents> {
        match self {
            Self::CompletedBody(events) => Some(events),
            Self::StreamingBody => None,
        }
    }
}

impl MainDocumentLiveNetworkProgressSource {
    pub(crate) fn from_navigation_dispatch_state(
        sender: Option<BackgroundEventSender>,
        session_ids: Vec<Option<String>>,
        state: &NavigationDispatchState,
        initial_request_cookie_report: Option<&StoredCookieQueryReport>,
    ) -> Option<Self> {
        Some(Self {
            sender,
            progress_queue: MainDocumentProgressQueueHandle::from_source(
                MainDocumentProgressSource::streaming(),
            ),
            session_ids,
            request_id: state.request_id.clone()?,
            loader_id: state.loader_id.clone(),
            frame_id: state.frame_id.clone(),
            timestamp: state.timestamp,
            initial_request_headers: state.request_headers.to_byte_strings(),
            initial_request_cookie_report: initial_request_cookie_report.cloned(),
        })
    }

    pub(crate) fn emit_initial_request_will_be_sent(
        &self,
        url: &Url,
        method: &str,
        request_body: Option<&str>,
        request_headers: &[(String, String)],
        cookie_access_report: Option<&StoredCookieQueryReport>,
    ) {
        let events = vec![MainDocumentNavigationProgressEvent::RequestWillBeSent {
            target: self.progress_target(),
            url: url.clone(),
            method: method.to_owned(),
            request_body: request_body.map(str::to_owned),
            request_headers: request_headers.to_vec(),
            request_initiator_type: SubresourceRequestInitiatorType::Other,
            redirect_response: Box::new(None),
            redirect_has_extra_info: false,
            cookie_access_report: cookie_access_report.cloned(),
        }];
        self.send_progress_events(MainDocumentProgressPhase::RequestStarted, events);
    }

    pub(crate) fn emit_redirect_requests(
        &self,
        final_request_method: &str,
        request_body: Option<&str>,
        final_request_headers: &[(String, String)],
        final_request_cookie_report: Option<&StoredCookieQueryReport>,
        redirect_chain: &[RedirectInfo],
        network_observation_journal: &NetworkObservationJournal,
        final_network_extra_info_available: bool,
    ) {
        let events = self.redirect_request_events(
            final_request_method,
            request_body,
            final_request_headers,
            final_request_cookie_report,
            redirect_chain,
            network_observation_journal,
            final_network_extra_info_available,
        );
        self.send_progress_events(MainDocumentProgressPhase::RequestStarted, events);
    }

    fn emit_redirect_requests_into_output(
        &self,
        output: &mut MainDocumentProgressOutputTarget<'_>,
        final_request_method: &str,
        request_body: Option<&str>,
        final_request_headers: &[(String, String)],
        final_request_cookie_report: Option<&StoredCookieQueryReport>,
        redirect_chain: &[RedirectInfo],
        network_observation_journal: &NetworkObservationJournal,
        final_network_extra_info_available: bool,
    ) {
        let events = self.redirect_request_events(
            final_request_method,
            request_body,
            final_request_headers,
            final_request_cookie_report,
            redirect_chain,
            network_observation_journal,
            final_network_extra_info_available,
        );
        self.send_progress_events_into_output(
            MainDocumentProgressPhase::RequestStarted,
            events,
            output,
        );
    }

    fn redirect_request_events(
        &self,
        final_request_method: &str,
        request_body: Option<&str>,
        final_request_headers: &[(String, String)],
        final_request_cookie_report: Option<&StoredCookieQueryReport>,
        redirect_chain: &[RedirectInfo],
        network_observation_journal: &NetworkObservationJournal,
        final_network_extra_info_available: bool,
    ) -> Vec<MainDocumentNavigationProgressEvent> {
        let mut events = Vec::new();
        let initial_network_extra_info_available = !network_observation_journal.is_empty()
            || redirect_chain
                .first()
                .is_some_and(|redirect| redirect.network_extra_info_available)
            || (redirect_chain.is_empty() && final_network_extra_info_available);
        if initial_network_extra_info_available {
            let initial_request_cookie_report = if redirect_chain.is_empty() {
                final_request_cookie_report
                    .cloned()
                    .or_else(|| self.initial_request_cookie_report.clone())
            } else {
                self.initial_request_cookie_report.clone()
            };
            events.push(request_extra_info_event(
                self.progress_target(),
                observed_request_headers(
                    network_observation_journal,
                    redirect_chain.len(),
                    0,
                    &self.initial_request_headers,
                ),
                initial_request_cookie_report,
            ));
        }
        for (index, redirect) in redirect_chain.iter().enumerate() {
            let target = self.progress_target();
            let observed_response_available =
                navigation_exchange_group(network_observation_journal, redirect_chain.len(), index)
                    .and_then(|group| group.last())
                    .and_then(NetworkExchangeObservation::response)
                    .is_some();
            if observed_response_available && !redirect.network_extra_info_available {
                let (status, headers) = observed_response_metadata(
                    network_observation_journal,
                    redirect_chain.len(),
                    index,
                    redirect.status,
                    &redirect.headers,
                );
                events.push(
                    MainDocumentNavigationProgressEvent::ResponseReceivedExtraInfo {
                        target: target.clone(),
                        headers,
                        status,
                        cookie_set_reports: redirect.cookie_set_reports.clone(),
                    },
                );
            }
            events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
                target: target.clone(),
                url: redirect.to_url.clone(),
                method: final_request_method.to_owned(),
                request_body: request_body.map(str::to_owned),
                request_headers: final_request_headers.to_vec(),
                request_initiator_type: SubresourceRequestInitiatorType::Other,
                redirect_response: Box::new(Some(MainDocumentRedirectResponse {
                    url: redirect.from_url.clone(),
                    status: redirect.status,
                    status_text: critical_client_hint_internal_redirect_status_text(
                        &redirect.from_url,
                        &redirect.to_url,
                        redirect.status,
                        redirect.network_extra_info_available,
                        observed_response_available,
                    ),
                    headers: redirect.headers.clone(),
                    from_cache: redirect.from_cache,
                    negotiated_http_version: redirect.negotiated_http_version,
                })),
                redirect_has_extra_info: redirect.network_extra_info_available,
                cookie_access_report: redirect.request_cookie_report.clone(),
            });
            if observed_response_available && redirect.network_extra_info_available {
                let (status, headers) = observed_response_metadata(
                    network_observation_journal,
                    redirect_chain.len(),
                    index,
                    redirect.status,
                    &redirect.headers,
                );
                events.push(
                    MainDocumentNavigationProgressEvent::ResponseReceivedExtraInfo {
                        target: target.clone(),
                        headers,
                        status,
                        cookie_set_reports: redirect.cookie_set_reports.clone(),
                    },
                );
            }
            let request_network_extra_info_available = network_observation_journal
                .exchanges()
                .get(index + 1)
                .is_some()
                || redirect_chain
                    .get(index + 1)
                    .is_some_and(|next| next.network_extra_info_available)
                || (index + 1 == redirect_chain.len() && final_network_extra_info_available);
            if request_network_extra_info_available {
                let cookie_report = redirect.request_cookie_report.clone().or_else(|| {
                    (index + 1 == redirect_chain.len())
                        .then(|| final_request_cookie_report.cloned())
                        .flatten()
                });
                events.push(request_extra_info_event(
                    target,
                    observed_request_headers(
                        network_observation_journal,
                        redirect_chain.len(),
                        index + 1,
                        final_request_headers,
                    ),
                    cookie_report,
                ));
            }
        }
        events
    }

    pub(crate) fn emit_response_received(
        &self,
        final_url: &Url,
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        response_cookie_reports: &[StoredCookieSetReport],
        network_observation_journal: &NetworkObservationJournal,
        redirect_count: usize,
        network_extra_info_available: bool,
        response_from_cache: bool,
        negotiated_http_version: Option<NegotiatedHttpVersion>,
    ) {
        let (extra_info_status, extra_info_headers) = observed_response_metadata(
            network_observation_journal,
            redirect_count,
            redirect_count,
            response_status,
            response_headers,
        );
        self.emit_response_received_with_extra_info_policy(
            final_url,
            response_status,
            response_headers,
            response_cookie_reports,
            extra_info_status,
            extra_info_headers,
            network_extra_info_available,
            network_extra_info_available,
            response_from_cache,
            negotiated_http_version,
            network_extra_info_available && !response_from_cache,
        );
    }

    fn emit_response_received_without_extra_info(
        &self,
        final_url: &Url,
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        network_extra_info_available: bool,
        response_from_cache: bool,
        negotiated_http_version: Option<NegotiatedHttpVersion>,
    ) {
        self.emit_response_received_with_extra_info_policy(
            final_url,
            response_status,
            response_headers,
            &[],
            response_status,
            response_headers.to_vec(),
            network_extra_info_available,
            false,
            response_from_cache,
            negotiated_http_version,
            network_extra_info_available && !response_from_cache,
        );
    }

    fn emit_response_received_with_extra_info_policy(
        &self,
        final_url: &Url,
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        response_cookie_reports: &[StoredCookieSetReport],
        extra_info_status: u16,
        extra_info_headers: Vec<(String, Vec<u8>)>,
        network_extra_info_available: bool,
        emit_extra_info: bool,
        response_from_cache: bool,
        negotiated_http_version: Option<NegotiatedHttpVersion>,
        has_extra_info: bool,
    ) {
        self.send_progress_events(
            MainDocumentProgressPhase::ResponseReceived,
            vec![MainDocumentNavigationProgressEvent::ResponseReceived {
                target: self.progress_target(),
                final_url: final_url.clone(),
                status: response_status,
                headers: response_headers.to_vec(),
                cookie_set_reports: response_cookie_reports.to_vec(),
                extra_info_status,
                extra_info_headers,
                network_extra_info_available,
                emit_extra_info,
                encoded_data_length: 0,
                from_cache: response_from_cache,
                negotiated_http_version,
                has_extra_info,
            }],
        );
    }

    fn emit_response_received_extra_info_into_output(
        &self,
        output: &mut MainDocumentProgressOutputTarget<'_>,
        response_status: u16,
        response_headers: &[(String, Vec<u8>)],
        response_cookie_reports: &[StoredCookieSetReport],
    ) {
        self.send_progress_events_into_output(
            MainDocumentProgressPhase::RequestStarted,
            vec![
                MainDocumentNavigationProgressEvent::ResponseReceivedExtraInfo {
                    target: self.progress_target(),
                    headers: response_headers.to_vec(),
                    status: response_status,
                    cookie_set_reports: response_cookie_reports.to_vec(),
                },
            ],
            output,
        );
    }

    pub(crate) fn emit_body_finished(&self, encoded_data_length: usize) {
        self.send_progress_events(
            MainDocumentProgressPhase::BodyFinished,
            vec![MainDocumentNavigationProgressEvent::LoadingFinished {
                target: self.progress_target(),
                encoded_data_length,
            }],
        );
    }

    fn progress_target(&self) -> MainDocumentProgressEventTarget {
        MainDocumentProgressEventTarget {
            session_ids: self.session_ids.clone(),
            request_id: self.request_id.clone(),
            loader_id: self.loader_id.clone(),
            frame_id: self.frame_id.clone(),
            timestamp: self.timestamp,
        }
    }

    fn send_progress_events(
        &self,
        ready_until: MainDocumentProgressPhase,
        events: Vec<MainDocumentNavigationProgressEvent>,
    ) {
        let Some(sender) = self.sender.as_ref() else {
            return;
        };
        let mut output = MainDocumentProgressOutputTarget::background_sender(sender);
        self.send_progress_events_into_output(ready_until, events, &mut output);
    }

    fn send_progress_events_into_output(
        &self,
        ready_until: MainDocumentProgressPhase,
        events: Vec<MainDocumentNavigationProgressEvent>,
        output: &mut MainDocumentProgressOutputTarget<'_>,
    ) {
        let emission = MainDocumentProgressEmission::new(
            ready_until,
            MainDocumentProgressEventBatch::from_events(events),
        );
        self.progress_queue
            .append_ready_emission(ready_until, emission);
        self.progress_queue.drain_into_output_target(output);
    }
}

impl std::fmt::Debug for MainDocumentLiveNetworkProgressSource {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("MainDocumentLiveNetworkProgressSource")
            .field("session_ids", &self.session_ids)
            .field("request_id", &self.request_id)
            .field("loader_id", &self.loader_id)
            .field("frame_id", &self.frame_id)
            .field("timestamp", &self.timestamp)
            .finish_non_exhaustive()
    }
}

fn emit_main_document_initial_request_will_be_sent_for_sessions_into(
    output: &mut MainDocumentProgressOutputTarget<'_>,
    session_ids: &[Option<String>],
    state: &NavigationDispatchState,
    cookie_access_report: Option<&StoredCookieQueryReport>,
) -> bool {
    let Some(request_id) = state.request_id.clone() else {
        return false;
    };
    let target = MainDocumentProgressEventTarget {
        session_ids: session_ids.to_vec(),
        request_id,
        loader_id: state.loader_id.clone(),
        frame_id: state.frame_id.clone(),
        timestamp: state.timestamp,
    };
    output.emit_event(MainDocumentNavigationProgressEvent::RequestWillBeSent {
        target,
        url: state.requested_url.clone(),
        method: state.request_method.clone(),
        request_body: state.request_body.clone(),
        request_headers: state.request_headers.to_byte_strings(),
        request_initiator_type: SubresourceRequestInitiatorType::Other,
        redirect_response: Box::new(None),
        redirect_has_extra_info: false,
        cookie_access_report: cookie_access_report.cloned(),
    });
    true
}

pub(crate) fn emit_fetch_navigation_initial_request_for_pause_background_events(
    conn: &CdpConnection,
    out: &mut Vec<BackgroundProtocolEvent>,
    state: &NavigationDispatchState,
    cookie_access_report: Option<&StoredCookieQueryReport>,
) -> bool {
    let mut session_ids =
        main_document_network_event_session_ids(conn, state.session_id.as_deref());
    if session_ids.is_empty() {
        session_ids.push(state.session_id.clone());
    }
    let mut output = MainDocumentProgressOutputTarget::background_events(out);
    emit_main_document_initial_request_will_be_sent_for_sessions_into(
        &mut output,
        &session_ids,
        state,
        cookie_access_report,
    )
}

pub(crate) fn emit_child_document_navigation_network_background_events(
    conn: &mut CdpConnection,
    out: &mut Vec<BackgroundProtocolEvent>,
    session_id: Option<&str>,
    frame_id: &str,
    loader_id: &str,
    request_id: &str,
    timestamp: f64,
    network: &moli_core::page::ChildFrameDocumentNetworkSnapshot,
) {
    let Ok(request_url) = Url::parse(&network.request_url) else {
        return;
    };
    let Ok(final_url) = Url::parse(&network.final_url) else {
        return;
    };
    let session_ids = conn.network_event_session_ids_for_session_owner(session_id);
    if session_ids.is_empty() {
        return;
    }
    let target = MainDocumentProgressEventTarget {
        session_ids: session_ids.clone(),
        request_id: request_id.to_owned(),
        loader_id: loader_id.to_owned(),
        frame_id: frame_id.to_owned(),
        timestamp,
    };
    let mut output = MainDocumentProgressOutputTarget::background_events(out);
    output.emit_event(MainDocumentNavigationProgressEvent::RequestWillBeSent {
        target: target.clone(),
        url: request_url,
        method: network.request_method.clone(),
        request_body: None,
        request_headers: network.request_headers.clone(),
        request_initiator_type: SubresourceRequestInitiatorType::Parser,
        redirect_response: Box::new(None),
        redirect_has_extra_info: false,
        cookie_access_report: None,
    });
    output.emit_event(MainDocumentNavigationProgressEvent::ResponseReceived {
        target: target.clone(),
        final_url,
        status: network.status,
        headers: network.response_headers.clone(),
        cookie_set_reports: Vec::new(),
        extra_info_status: network.status,
        extra_info_headers: network.response_headers.clone(),
        network_extra_info_available: false,
        emit_extra_info: false,
        encoded_data_length: 0,
        from_cache: network.from_cache,
        negotiated_http_version: None,
        has_extra_info: false,
    });
    record_child_document_response_body(
        conn,
        session_id,
        request_id,
        &session_ids,
        network.response_body.as_ref(),
    );
    output.emit_event(MainDocumentNavigationProgressEvent::LoadingFinished {
        target,
        encoded_data_length: network.encoded_data_length,
    });
}

fn record_child_document_response_body(
    conn: &mut CdpConnection,
    owner_session_id: Option<&str>,
    request_id: &str,
    session_ids: &[Option<String>],
    response_body: Option<&moli_core::page::SubresourceResponseBody>,
) {
    let data_type = crate::automation::DevToolsNetworkDataType::Response;
    let encoded_data_length = response_body.map_or(0, |body| body.len());
    let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
        owner_session_id,
        data_type,
        encoded_data_length,
    );
    let collection_was_gated = conn.network_data_collection_is_gated_for_body(data_type);
    let captured_body =
        response_body.map(crate::conn::CapturedBody::from_subresource_response_body);
    if let Some(captured_body) = captured_body.as_ref() {
        conn.record_collected_network_data_body(
            request_id.to_owned(),
            data_type,
            captured_body.clone(),
            collector_ids.iter().cloned(),
            collection_was_gated,
        );
    }
    let Some(runtime_slot) = conn.runtime_session_owner_slot_mut(owner_session_id) else {
        return;
    };
    if let Some(captured_body) = captured_body {
        runtime_slot.record_captured_response_body_source_with_collector_scope(
            request_id.to_owned(),
            captured_body,
            session_ids.iter().cloned(),
            collector_ids,
            collection_was_gated,
        );
    } else {
        runtime_slot.record_failed_response_body_with_collector_scope(
            request_id.to_owned(),
            "child document response body unavailable".to_owned(),
            session_ids.iter().cloned(),
            collector_ids,
            collection_was_gated,
        );
    }
}

pub(crate) fn start_observed_main_document_navigation_progress_background_events(
    conn: &mut CdpConnection,
    out: &mut Vec<BackgroundProtocolEvent>,
    state: &NavigationDispatchState,
    cookie_access_report: Option<&StoredCookieQueryReport>,
) -> MainDocumentBodyProgressSource {
    let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
    record_pending_main_document_response_body(conn, state, &session_ids);
    record_main_document_request_body(conn, state);
    if let Some(sender) = conn.background_event_sender()
        && let Some(live_source) =
            MainDocumentLiveNetworkProgressSource::from_navigation_dispatch_state(
                Some(sender),
                session_ids.clone(),
                state,
                cookie_access_report,
            )
    {
        live_source.emit_initial_request_will_be_sent(
            &state.requested_url,
            &state.request_method,
            state.request_body.as_deref(),
            &state.request_headers.to_byte_strings(),
            cookie_access_report,
        );
        return MainDocumentBodyProgressSource::from_live_source(
            Some(live_source),
            MainDocumentResponseVisibility::Immediate,
        );
    }
    let mut output = MainDocumentProgressOutputTarget::background_events(out);
    emit_main_document_initial_request_will_be_sent_for_sessions_into(
        &mut output,
        &session_ids,
        state,
        cookie_access_report,
    );
    MainDocumentBodyProgressSource::default()
}

pub(crate) fn response_stage_main_document_navigation_network_progress(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    initial_request_cookie_report: Option<&StoredCookieQueryReport>,
) -> MainDocumentBodyProgressSource {
    if !main_document_network_observed(conn, state.session_id.as_deref()) {
        return MainDocumentBodyProgressSource::default();
    }
    let live_source = MainDocumentLiveNetworkProgressSource::from_navigation_dispatch_state(
        conn.background_event_sender(),
        main_document_network_event_session_ids(conn, state.session_id.as_deref()),
        state,
        initial_request_cookie_report,
    );
    MainDocumentBodyProgressSource::from_live_source(
        live_source,
        MainDocumentResponseVisibility::AfterResponseStageContinue,
    )
}

fn observed_navigation_failure_event(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
    error_text: &str,
) -> Option<MainDocumentNavigationProgressEvent> {
    if !conn.has_network_event_listeners_for_session_owner(state.session_id.as_deref()) {
        return None;
    }
    let request_id = state.request_id.clone()?;
    Some(MainDocumentNavigationProgressEvent::LoadingFailed {
        target: MainDocumentProgressEventTarget {
            session_ids: conn
                .network_event_session_ids_for_session_owner(state.session_id.as_deref()),
            request_id,
            loader_id: state.loader_id.clone(),
            frame_id: state.frame_id.clone(),
            timestamp: state.timestamp,
        },
        error_text: error_text.to_owned(),
    })
}

fn observed_navigation_finished_event(
    conn: &CdpConnection,
    state: &NavigationDispatchState,
) -> Option<MainDocumentNavigationProgressEvent> {
    if !conn.has_network_event_listeners_for_session_owner(state.session_id.as_deref()) {
        return None;
    }
    let request_id = state.request_id.clone()?;
    Some(MainDocumentNavigationProgressEvent::LoadingFinished {
        target: MainDocumentProgressEventTarget {
            session_ids: conn
                .network_event_session_ids_for_session_owner(state.session_id.as_deref()),
            request_id,
            loader_id: state.loader_id.clone(),
            frame_id: state.frame_id.clone(),
            timestamp: state.timestamp,
        },
        encoded_data_length: 0,
    })
}

fn main_document_network_event_session_ids(
    conn: &CdpConnection,
    trigger_session_id: Option<&str>,
) -> Vec<Option<String>> {
    conn.network_event_session_ids_for_session_owner(trigger_session_id)
}

struct CompletedMainDocumentProgressContext {
    session_ids: Vec<Option<String>>,
    request_id: Option<String>,
    request_announced: bool,
    requested_url: Url,
    request_method: String,
    request_body: Option<String>,
    request_headers: Vec<(String, String)>,
    loader_id: String,
    frame_id: String,
    timestamp: f64,
}

impl CompletedMainDocumentProgressContext {
    fn for_navigation(
        conn: &CdpConnection,
        state: &NavigationDispatchState,
        request_id: Option<String>,
    ) -> Self {
        Self::new(
            main_document_network_event_session_ids(conn, state.session_id.as_deref()),
            request_id,
            state.request_announced,
            state.requested_url.clone(),
            state.request_method.clone(),
            state.request_body.clone(),
            state.request_headers.to_byte_strings(),
            state.loader_id.clone(),
            state.frame_id.clone(),
            state.timestamp,
        )
    }

    fn new(
        session_ids: Vec<Option<String>>,
        request_id: Option<String>,
        request_announced: bool,
        requested_url: Url,
        request_method: String,
        request_body: Option<String>,
        request_headers: Vec<(String, String)>,
        loader_id: String,
        frame_id: String,
        timestamp: f64,
    ) -> Self {
        Self {
            session_ids,
            request_id,
            request_announced,
            requested_url,
            request_method,
            request_body,
            request_headers,
            loader_id,
            frame_id,
            timestamp,
        }
    }

    fn event_batches(
        &self,
        events: &CompletedMainDocumentNetworkEvents,
        final_url: &Url,
        encoded_data_length: usize,
    ) -> MainDocumentNavigationProgressEventBatches {
        MainDocumentNavigationProgressEventBatches::new(
            self.request_and_redirect_progress_events(events),
            self.response_received_progress_events(events, final_url, encoded_data_length),
            self.loading_finished_progress_events(encoded_data_length),
        )
    }

    fn request_and_redirect_progress_events(
        &self,
        events: &CompletedMainDocumentNetworkEvents,
    ) -> Vec<MainDocumentNavigationProgressEvent> {
        if events.response_stage_metadata_already_emitted {
            return Vec::new();
        }
        let Some(target) = self.progress_target() else {
            return Vec::new();
        };
        let mut progress_events = Vec::new();
        let initial_request_cookie_report = events
            .redirect_chain
            .is_empty()
            .then(|| events.final_request_cookie_report.clone())
            .flatten();
        if !self.request_announced {
            progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
                target: target.clone(),
                url: self.requested_url.clone(),
                method: self.request_method.clone(),
                request_body: self.request_body.clone(),
                request_headers: self.request_headers.clone(),
                request_initiator_type: SubresourceRequestInitiatorType::Other,
                redirect_response: Box::new(None),
                redirect_has_extra_info: false,
                cookie_access_report: initial_request_cookie_report.clone(),
            });
        }
        let initial_network_extra_info_available = !events.network_observation_journal.is_empty()
            || events
                .redirect_chain
                .first()
                .is_some_and(|redirect| redirect.network_extra_info_available)
            || (events.redirect_chain.is_empty() && events.network_extra_info_available);
        if initial_network_extra_info_available {
            progress_events.push(request_extra_info_event(
                target.clone(),
                observed_request_headers(
                    &events.network_observation_journal,
                    events.redirect_chain.len(),
                    0,
                    &self.request_headers,
                ),
                initial_request_cookie_report,
            ));
        }
        for (index, redirect) in events.redirect_chain.iter().enumerate() {
            let observed_response_available = navigation_exchange_group(
                &events.network_observation_journal,
                events.redirect_chain.len(),
                index,
            )
            .and_then(|group| group.last())
            .and_then(NetworkExchangeObservation::response)
            .is_some();
            if observed_response_available && !redirect.network_extra_info_available {
                let (status, headers) = observed_response_metadata(
                    &events.network_observation_journal,
                    events.redirect_chain.len(),
                    index,
                    redirect.status,
                    &redirect.headers,
                );
                progress_events.push(
                    MainDocumentNavigationProgressEvent::ResponseReceivedExtraInfo {
                        target: target.clone(),
                        headers,
                        status,
                        cookie_set_reports: redirect.cookie_set_reports.clone(),
                    },
                );
            }
            progress_events.push(MainDocumentNavigationProgressEvent::RequestWillBeSent {
                target: target.clone(),
                url: redirect.to_url.clone(),
                method: events.request_method.clone(),
                request_body: self.request_body.clone(),
                request_headers: events.request_headers.clone(),
                request_initiator_type: SubresourceRequestInitiatorType::Other,
                redirect_response: Box::new(Some(MainDocumentRedirectResponse {
                    url: redirect.from_url.clone(),
                    status: redirect.status,
                    status_text: critical_client_hint_internal_redirect_status_text(
                        &redirect.from_url,
                        &redirect.to_url,
                        redirect.status,
                        redirect.network_extra_info_available,
                        observed_response_available,
                    ),
                    headers: redirect.headers.clone(),
                    from_cache: redirect.from_cache,
                    negotiated_http_version: redirect.negotiated_http_version,
                })),
                redirect_has_extra_info: redirect.network_extra_info_available,
                cookie_access_report: redirect.request_cookie_report.clone(),
            });
            if observed_response_available && redirect.network_extra_info_available {
                let (status, headers) = observed_response_metadata(
                    &events.network_observation_journal,
                    events.redirect_chain.len(),
                    index,
                    redirect.status,
                    &redirect.headers,
                );
                progress_events.push(
                    MainDocumentNavigationProgressEvent::ResponseReceivedExtraInfo {
                        target: target.clone(),
                        headers,
                        status,
                        cookie_set_reports: redirect.cookie_set_reports.clone(),
                    },
                );
            }
            let request_network_extra_info_available = events
                .network_observation_journal
                .exchanges()
                .get(index + 1)
                .is_some()
                || events
                    .redirect_chain
                    .get(index + 1)
                    .is_some_and(|next| next.network_extra_info_available)
                || (index + 1 == events.redirect_chain.len()
                    && events.network_extra_info_available);
            if request_network_extra_info_available {
                let cookie_report = redirect.request_cookie_report.clone().or_else(|| {
                    (index + 1 == events.redirect_chain.len())
                        .then(|| events.final_request_cookie_report.clone())
                        .flatten()
                });
                progress_events.push(request_extra_info_event(
                    target.clone(),
                    observed_request_headers(
                        &events.network_observation_journal,
                        events.redirect_chain.len(),
                        index + 1,
                        &events.request_headers,
                    ),
                    cookie_report,
                ));
            }
        }
        progress_events
    }

    fn response_received_progress_events(
        &self,
        events: &CompletedMainDocumentNetworkEvents,
        final_url: &Url,
        encoded_data_length: usize,
    ) -> Vec<MainDocumentNavigationProgressEvent> {
        let Some(target) = self.progress_target() else {
            return Vec::new();
        };
        let (extra_info_status, extra_info_headers) = observed_response_metadata(
            &events.network_observation_journal,
            events.redirect_chain.len(),
            events.redirect_chain.len(),
            events.response_status,
            &events.response_headers,
        );
        vec![MainDocumentNavigationProgressEvent::ResponseReceived {
            target,
            final_url: final_url.clone(),
            status: events.response_status,
            headers: events.response_headers.clone(),
            cookie_set_reports: events.response_cookie_reports.clone(),
            extra_info_status,
            extra_info_headers,
            network_extra_info_available: events.network_extra_info_available,
            emit_extra_info: events.network_extra_info_available
                && !events.response_stage_metadata_already_emitted,
            encoded_data_length,
            from_cache: events.response_from_cache,
            negotiated_http_version: events.negotiated_http_version,
            has_extra_info: events.network_extra_info_available && !events.response_from_cache,
        }]
    }

    fn loading_finished_progress_events(
        &self,
        encoded_data_length: usize,
    ) -> Vec<MainDocumentNavigationProgressEvent> {
        let Some(target) = self.progress_target() else {
            return Vec::new();
        };
        vec![MainDocumentNavigationProgressEvent::LoadingFinished {
            target,
            encoded_data_length,
        }]
    }

    fn progress_target(&self) -> Option<MainDocumentProgressEventTarget> {
        Some(MainDocumentProgressEventTarget {
            session_ids: self.session_ids.clone(),
            request_id: self.request_id.clone()?,
            loader_id: self.loader_id.clone(),
            frame_id: self.frame_id.clone(),
            timestamp: self.timestamp,
        })
    }
}

struct MainDocumentNavigationProgressEventBatches {
    request_started: Vec<MainDocumentNavigationProgressEvent>,
    response_received: Vec<MainDocumentNavigationProgressEvent>,
    body_finished: Vec<MainDocumentNavigationProgressEvent>,
}

#[derive(Clone)]
pub(crate) struct MainDocumentProgressEventTarget {
    pub(crate) session_ids: Vec<Option<String>>,
    pub(crate) request_id: String,
    pub(crate) loader_id: String,
    pub(crate) frame_id: String,
    pub(crate) timestamp: f64,
}

enum MainDocumentNavigationProgressEvent {
    RequestWillBeSent {
        target: MainDocumentProgressEventTarget,
        url: Url,
        method: String,
        request_body: Option<String>,
        request_headers: Vec<(String, String)>,
        request_initiator_type: SubresourceRequestInitiatorType,
        redirect_response: Box<Option<MainDocumentRedirectResponse>>,
        redirect_has_extra_info: bool,
        cookie_access_report: Option<StoredCookieQueryReport>,
    },
    RequestWillBeSentExtraInfo {
        target: MainDocumentProgressEventTarget,
        request_headers: Vec<(String, String)>,
        cookie_access_report: StoredCookieQueryReport,
    },
    ResponseReceivedExtraInfo {
        target: MainDocumentProgressEventTarget,
        headers: Vec<(String, Vec<u8>)>,
        status: u16,
        cookie_set_reports: Vec<StoredCookieSetReport>,
    },
    ResponseReceived {
        target: MainDocumentProgressEventTarget,
        final_url: Url,
        status: u16,
        headers: Vec<(String, Vec<u8>)>,
        cookie_set_reports: Vec<StoredCookieSetReport>,
        extra_info_status: u16,
        extra_info_headers: Vec<(String, Vec<u8>)>,
        network_extra_info_available: bool,
        emit_extra_info: bool,
        encoded_data_length: usize,
        from_cache: bool,
        negotiated_http_version: Option<NegotiatedHttpVersion>,
        has_extra_info: bool,
    },
    LoadingFinished {
        target: MainDocumentProgressEventTarget,
        encoded_data_length: usize,
    },
    LoadingFailed {
        target: MainDocumentProgressEventTarget,
        error_text: String,
    },
}

pub(crate) struct MainDocumentRedirectResponse {
    pub(crate) url: Url,
    pub(crate) status: u16,
    pub(crate) status_text: Option<String>,
    pub(crate) headers: Vec<(String, Vec<u8>)>,
    pub(crate) from_cache: bool,
    pub(crate) negotiated_http_version: Option<NegotiatedHttpVersion>,
}

fn critical_client_hint_internal_redirect_status_text(
    from_url: &Url,
    to_url: &Url,
    status: u16,
    redirect_has_extra_info: bool,
    discarded_network_response_has_extra_info: bool,
) -> Option<String> {
    (status == 307
        && from_url == to_url
        && !redirect_has_extra_info
        && discarded_network_response_has_extra_info)
        .then(|| "Internal Redirect".to_owned())
}

fn request_extra_info_event(
    target: MainDocumentProgressEventTarget,
    request_headers: Vec<(String, String)>,
    cookie_access_report: Option<StoredCookieQueryReport>,
) -> MainDocumentNavigationProgressEvent {
    MainDocumentNavigationProgressEvent::RequestWillBeSentExtraInfo {
        target,
        request_headers,
        cookie_access_report: cookie_access_report.unwrap_or_default(),
    }
}

fn navigation_exchange_group(
    journal: &NetworkObservationJournal,
    redirect_count: usize,
    hop_index: usize,
) -> Option<&[NetworkExchangeObservation]> {
    // A truncated journal still proves that a transport exchange happened, but
    // no longer provides a trustworthy tail for redirect-hop correlation.
    if journal.truncated() {
        return None;
    }
    if hop_index > redirect_count {
        return None;
    }

    let exchanges = journal.exchanges();
    let mut group_start = 0;
    let mut current_hop = 0;
    for (index, exchange) in exchanges.iter().enumerate() {
        let ends_redirect_hop = exchange.response().is_some_and(|response| {
            matches!(response.status(), 301 | 302 | 303 | 307 | 308)
                || response.headers().iter().any(|(name, value)| {
                    name.eq_ignore_ascii_case("critical-ch") && !value.trim_ascii().is_empty()
                })
        });
        if !ends_redirect_hop || current_hop >= redirect_count {
            continue;
        }
        if current_hop == hop_index {
            return Some(&exchanges[group_start..=index]);
        }
        current_hop += 1;
        group_start = index.saturating_add(1);
    }
    (current_hop == hop_index && group_start < exchanges.len()).then_some(&exchanges[group_start..])
}

fn observed_request_headers(
    journal: &NetworkObservationJournal,
    redirect_count: usize,
    hop_index: usize,
    fallback: &[(String, String)],
) -> Vec<(String, String)> {
    navigation_exchange_group(journal, redirect_count, hop_index)
        .and_then(|group| group.first())
        .map(|exchange| exchange.request().headers().to_vec())
        .unwrap_or_else(|| fallback.to_vec())
}

fn observed_response_metadata(
    journal: &NetworkObservationJournal,
    redirect_count: usize,
    hop_index: usize,
    fallback_status: u16,
    fallback_headers: &[(String, Vec<u8>)],
) -> (u16, Vec<(String, Vec<u8>)>) {
    navigation_exchange_group(journal, redirect_count, hop_index)
        .and_then(|group| group.last())
        .and_then(NetworkExchangeObservation::response)
        .map(|response| (response.status(), response.headers().to_vec()))
        .unwrap_or_else(|| (fallback_status, fallback_headers.to_vec()))
}

impl MainDocumentNavigationProgressEventBatches {
    fn new(
        request_started: Vec<MainDocumentNavigationProgressEvent>,
        response_received: Vec<MainDocumentNavigationProgressEvent>,
        body_finished: Vec<MainDocumentNavigationProgressEvent>,
    ) -> Self {
        Self {
            request_started,
            response_received,
            body_finished,
        }
    }

    fn take_request_started(&mut self) -> Vec<MainDocumentNavigationProgressEvent> {
        std::mem::take(&mut self.request_started)
    }

    fn take_response_received(&mut self) -> Vec<MainDocumentNavigationProgressEvent> {
        std::mem::take(&mut self.response_received)
    }

    fn take_body_finished(&mut self) -> Vec<MainDocumentNavigationProgressEvent> {
        std::mem::take(&mut self.body_finished)
    }
}

impl MainDocumentNavigationProgressEvent {
    fn emit_into(self, output: &mut MainDocumentProgressOutputTarget<'_>) {
        match self {
            Self::RequestWillBeSent {
                target,
                url,
                method,
                request_body,
                request_headers,
                request_initiator_type,
                redirect_response,
                redirect_has_extra_info,
                cookie_access_report,
            } => {
                let redirect_response = redirect_response.as_ref().as_ref().map(|redirect| {
                    (
                        &redirect.url,
                        redirect.status,
                        redirect.status_text.as_deref(),
                        redirect.headers.as_slice(),
                        redirect.from_cache,
                        redirect.negotiated_http_version,
                    )
                });
                for session_id in &target.session_ids {
                    emit::emit_main_document_request_will_be_sent(
                        output,
                        session_id.as_deref(),
                        &target.request_id,
                        &target.frame_id,
                        &target.loader_id,
                        target.timestamp,
                        &url,
                        &method,
                        request_body.as_deref(),
                        &request_headers,
                        request_initiator_type,
                        redirect_response,
                        redirect_has_extra_info,
                        cookie_access_report.as_ref(),
                    );
                }
            }
            Self::RequestWillBeSentExtraInfo {
                target,
                request_headers,
                cookie_access_report,
            } => {
                for session_id in &target.session_ids {
                    emit::emit_request_will_be_sent_extra_info(
                        output,
                        session_id.as_deref(),
                        &target.request_id,
                        &request_headers,
                        &cookie_access_report,
                        target.timestamp,
                    );
                }
            }
            Self::ResponseReceivedExtraInfo {
                target,
                headers,
                status,
                cookie_set_reports,
            } => {
                for session_id in &target.session_ids {
                    emit::emit_response_received_extra_info(
                        output,
                        session_id.as_deref(),
                        &target.request_id,
                        &headers,
                        status,
                        &cookie_set_reports,
                    );
                }
            }
            Self::ResponseReceived {
                target,
                final_url,
                status,
                headers,
                cookie_set_reports,
                extra_info_status,
                extra_info_headers,
                network_extra_info_available,
                emit_extra_info,
                encoded_data_length,
                from_cache,
                negotiated_http_version,
                has_extra_info,
            } => {
                for session_id in &target.session_ids {
                    emit::emit_main_document_response_received(
                        output,
                        session_id.as_deref(),
                        &target.request_id,
                        &target.frame_id,
                        &target.loader_id,
                        target.timestamp,
                        &final_url,
                        status,
                        &headers,
                        &cookie_set_reports,
                        extra_info_status,
                        &extra_info_headers,
                        network_extra_info_available,
                        emit_extra_info,
                        encoded_data_length,
                        from_cache,
                        negotiated_http_version,
                        has_extra_info,
                    );
                }
            }
            Self::LoadingFinished {
                target,
                encoded_data_length,
            } => {
                for session_id in &target.session_ids {
                    emit::emit_body_finished(
                        output,
                        session_id.as_deref(),
                        &target.request_id,
                        &target.frame_id,
                        &target.loader_id,
                        target.timestamp,
                        encoded_data_length,
                    );
                }
            }
            Self::LoadingFailed { target, error_text } => {
                for session_id in &target.session_ids {
                    emit::emit_loading_failed(
                        output,
                        session_id.as_deref(),
                        &target.request_id,
                        &target.frame_id,
                        &target.loader_id,
                        target.timestamp,
                        &error_text,
                    );
                }
            }
        }
    }
}

fn completed_body_main_document_network_request_id(
    network_enabled: bool,
    request_id: Option<String>,
) -> Option<String> {
    if network_enabled { request_id } else { None }
}

fn record_main_document_response_body_for_network(
    runtime_slot: &mut TargetRuntimeSlot,
    network_enabled: bool,
    request_id: Option<String>,
    session_ids: &[Option<String>],
    collector_ids: Vec<String>,
    collection_was_gated: bool,
    synthetic: bool,
    response_body: &CapturedBody,
) -> Option<String> {
    if !network_enabled || synthetic {
        return None;
    }
    let request_id = request_id?;
    runtime_slot.record_captured_response_body_source_with_collector_scope(
        request_id.clone(),
        response_body.clone(),
        session_ids.iter().cloned(),
        collector_ids,
        collection_was_gated,
    );
    Some(request_id)
}

fn record_pending_main_document_response_body(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    session_ids: &[Option<String>],
) {
    if !main_document_network_observed(conn, state.session_id.as_deref()) {
        return;
    }
    let Some(request_id) = state.request_id.clone() else {
        return;
    };
    let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
        state.session_id.as_deref(),
        crate::automation::DevToolsNetworkDataType::Response,
        0,
    );
    let collection_was_gated = conn.network_data_collection_is_gated_for_body(
        crate::automation::DevToolsNetworkDataType::Response,
    );
    if let Some(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {
        runtime_slot.record_pending_response_body_with_collector_scope(
            request_id,
            session_ids.iter().cloned(),
            collector_ids,
            collection_was_gated,
        );
    }
}

pub(crate) fn record_main_document_request_body(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
) {
    if !main_document_network_observed(conn, state.session_id.as_deref()) {
        return;
    }
    let Some(request_id) = state.request_id.clone() else {
        return;
    };
    let Some(transport_bytes) = state.clone_request_body_bytes() else {
        return;
    };
    let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
    let data_type = crate::automation::DevToolsNetworkDataType::Request;
    let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
        state.session_id.as_deref(),
        data_type,
        transport_bytes.len(),
    );
    let collection_was_gated = conn.network_data_collection_is_gated_for_body(data_type);
    // The BiDi network data collector keeps the raw transport bytes, including
    // multipart file payloads.
    conn.record_collected_network_data_body(
        request_id.clone(),
        data_type,
        crate::conn::CapturedBody::from_bytes(transport_bytes.clone()),
        collector_ids.iter().cloned(),
        collection_was_gated,
    );
    if let Some(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {
        // CDP getRequestPostData uses the text projection that request events
        // and Fetch interception expose, which omits multipart file contents.
        // Never surface raw transport bytes through the CDP captured store.
        let cdp_request_body = state
            .request_body
            .clone()
            .unwrap_or_else(|| String::from_utf8_lossy(&transport_bytes).into_owned());
        runtime_slot.record_captured_request_body_with_collector_scope(
            request_id,
            cdp_request_body.into(),
            session_ids.iter().cloned(),
            collector_ids,
            collection_was_gated,
        );
    }
}

pub(crate) fn record_failed_main_document_response_body(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    error_text: String,
) {
    if !main_document_network_observed(conn, state.session_id.as_deref()) {
        return;
    }
    let Some(request_id) = state.request_id.clone() else {
        return;
    };
    let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
    let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
        state.session_id.as_deref(),
        crate::automation::DevToolsNetworkDataType::Response,
        0,
    );
    let collection_was_gated = conn.network_data_collection_is_gated_for_body(
        crate::automation::DevToolsNetworkDataType::Response,
    );
    if let Some(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {
        runtime_slot.record_failed_response_body_with_collector_scope(
            request_id,
            error_text,
            session_ids,
            collector_ids,
            collection_was_gated,
        );
    }
}

pub(crate) fn record_completed_main_document_response_body(
    conn: &mut CdpConnection,
    state: &NavigationDispatchState,
    synthetic: bool,
    response_body: &CapturedBody,
) {
    let network_enabled = main_document_network_observed(conn, state.session_id.as_deref());
    let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
    let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
        state.session_id.as_deref(),
        crate::automation::DevToolsNetworkDataType::Response,
        response_body.len(),
    );
    let collection_was_gated = conn.network_data_collection_is_gated_for_body(
        crate::automation::DevToolsNetworkDataType::Response,
    );
    if network_enabled
        && !synthetic
        && let Some(request_id) = state.request_id.clone()
    {
        conn.record_collected_network_data_body(
            request_id,
            crate::automation::DevToolsNetworkDataType::Response,
            response_body.clone(),
            collector_ids.iter().cloned(),
            collection_was_gated,
        );
    }
    if let Some(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {
        let _ = record_main_document_response_body_for_network(
            runtime_slot,
            network_enabled,
            state.request_id.clone(),
            &session_ids,
            collector_ids,
            collection_was_gated,
            synthetic,
            response_body,
        );
    }
}

fn main_document_network_observed(conn: &CdpConnection, session_id: Option<&str>) -> bool {
    conn.has_network_event_listeners_for_session_owner(session_id)
}