Skip to content

The design language that makes your AI harness better at design.

JavaScript
15.7 KB
Raw Blame
//! Opt-in, bounded CDP response evidence. This proves observed bytes, not paint.
use base64::Engine;
use serde_json::Value;
use std::collections::{HashMap, HashSet};

pub(crate) const MAX_BODY: usize = 16 * 1024 * 1024;
pub(crate) const MAX_TOTAL: usize = 64 * 1024 * 1024;
const MAX_RECORDS: usize = 512;

#[derive(Debug, Clone)]
pub struct ResponseRecord {
    pub request_id: String,
    pub url: String,
    pub frame_id: String,
    pub loader_id: String,
    pub status: Option<f64>,
    pub mime_type: String,
    pub from_disk_cache: bool,
    pub from_service_worker: bool,
    pub complete: bool,
    /// CDP-decoded response payload, not HTTP transfer/compression bytes.
    pub body: Option<Vec<u8>>,
    /// CDP returned the renderer's decoded text (UTF-8), not the served bytes.
    /// Compare with `body_matches`, never `body ==` a frozen file.
    pub text: bool,
    pub unavailable_reason: Option<String>,
    /// Multiple observed requests for this URL; DOM URL alone cannot bind one.
    pub ambiguous_url: bool,
}

#[derive(Debug)]
pub struct ResponseEvidence {
    pub responses: Vec<ResponseRecord>,
    pub missing_urls: Vec<String>,
    /// Evidence is incomplete when the bounded event journal overflowed.
    pub truncated: bool,
    /// Network changed while retrieving bodies; callers must capture again.
    pub changed_during_collection: bool,
    pub revision: u64,
}

#[derive(Default)]
pub(crate) struct ResponseCapture {
    records: Vec<ResponseRecord>,
    active: HashMap<String, usize>,
    /// Requests the browser made for itself (the tab icon), never the page.
    browser_initiated: HashSet<String>,
    pub revision: u64,
    truncated: bool,
    body_bytes: usize,
}

/// The text Blink decodes from bytes served as `charset=utf-8`: a BOM wins over
/// the label, and invalid UTF-8 becomes U+FFFD (WHATWG maximal subparts, as
/// `from_utf8_lossy`). Other labels are not modelled, so they fail closed.
pub fn decoded_text(raw: &[u8]) -> Vec<u8> {
    let utf16 = |b: &[u8], be: bool| {
        let units: Vec<u16> = b.chunks(2).map(|c| match c {
            [x, y] => if be { u16::from_be_bytes([*x, *y]) } else { u16::from_le_bytes([*x, *y]) },
            _ => 0xFFFD,
        }).collect();
        String::from_utf16_lossy(&units).into_bytes()
    };
    match raw {
        [0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest).into_owned().into_bytes(),
        [0xFE, 0xFF, rest @ ..] => utf16(rest, true),
        [0xFF, 0xFE, rest @ ..] => utf16(rest, false),
        _ => String::from_utf8_lossy(raw).into_owned().into_bytes(),
    }
}

impl ResponseRecord {
    /// Whether the observed response is exactly `raw`: byte-for-byte when CDP
    /// returned bytes, or as the renderer's decoded text when it returned text
    /// (the only form of a text body the page consumes).
    pub fn body_matches(&self, raw: &[u8]) -> bool {
        self.body.as_deref().is_some_and(|b| if self.text { b == decoded_text(raw) } else { b == raw })
    }
}

fn string(v: &Value, key: &str) -> String {
    v.get(key).and_then(Value::as_str).unwrap_or("").to_owned()
}

/// The browser's own fetch for the tab icon: no parser, script or preload
/// initiated it and it is not a navigation, so CDP reports it as type `Other`
/// with an `other` initiator. Every request the document makes carries its own
/// type or initiator (a `<link rel=prefetch>` is `Other` from the `parser`).
/// Chrome sends it after the load event, on its own schedule, and it paints
/// nothing, so it is neither a page dependency nor a change to the page.
fn browser_initiated(p: &Value) -> bool {
    string(p, "type") == "Other"
        && p.pointer("/initiator/type").and_then(Value::as_str) == Some("other")
        && string(p, "requestId") != string(p, "loaderId")
}

impl ResponseCapture {
    pub fn event(&mut self, method: &str, p: &Value) {
        if !matches!(
            method,
            "Network.requestWillBeSent"
                | "Network.responseReceived"
                | "Network.loadingFinished"
                | "Network.loadingFailed"
        ) {
            return;
        }
        let id = string(p, "requestId");
        if self.browser_initiated.contains(&id) {
            return;
        }
        if method == "Network.requestWillBeSent" && !id.is_empty() && browser_initiated(p) {
            if self.browser_initiated.len() < MAX_RECORDS {
                self.browser_initiated.insert(id);
                return;
            }
            self.truncated = true;
        }
        self.revision += 1;
        if id.is_empty() {
            self.truncated = true;
            return;
        }
        if method == "Network.requestWillBeSent" {
            // Redirect hops reuse a request id. Never retrieve the final body's
            // bytes on behalf of an earlier hop, even when the URL repeats.
            if let Some(old) = self.active.remove(&id) {
                self.records[old].unavailable_reason =
                    Some("request id reused or redirected".into());
            }
            if self.records.len() >= MAX_RECORDS {
                self.truncated = true;
                return;
            }
            let index = self.records.len();
            self.records.push(ResponseRecord {
                request_id: id.clone(),
                url: string(&p["request"], "url"),
                frame_id: string(p, "frameId"),
                loader_id: string(p, "loaderId"),
                status: None,
                mime_type: String::new(),
                from_disk_cache: false,
                from_service_worker: false,
                complete: false,
                body: None,
                text: false,
                unavailable_reason: None,
                ambiguous_url: false,
            });
            self.active.insert(id, index);
            return;
        }
        let Some(&i) = self.active.get(&id) else {
            return;
        };
        let r = &mut self.records[i];
        match method {
            "Network.responseReceived" => {
                let response = &p["response"];
                r.status = response["status"].as_f64();
                r.mime_type = string(response, "mimeType");
                r.from_disk_cache = response["fromDiskCache"].as_bool().unwrap_or(false);
                r.from_service_worker = response["fromServiceWorker"].as_bool().unwrap_or(false);
                if string(response, "url") != r.url {
                    r.unavailable_reason = Some("response URL differs from request".into());
                }
            }
            "Network.loadingFinished" => r.complete = true,
            "Network.loadingFailed" => {
                r.unavailable_reason = Some(format!("request failed: {}", string(p, "errorText")))
            }
            _ => {}
        }
    }

    fn matches(r: &ResponseRecord, urls: &[String], frame: &str, loader: &str) -> bool {
        !frame.is_empty()
            && !loader.is_empty()
            && r.frame_id == frame
            && r.loader_id == loader
            && urls.contains(&r.url)
    }

    pub fn urls(&self, frame: &str, loader: &str) -> Vec<String> {
        let mut urls: Vec<_> = self.records.iter().filter(|r|r.frame_id==frame && r.loader_id==loader).map(|r|r.url.clone()).collect();
        urls.sort(); urls.dedup(); urls
    }

    pub fn pending(&self, urls: &[String], frame: &str, loader: &str) -> Vec<(usize, String)> {
        self.records
            .iter()
            .enumerate()
            .filter(|(_, r)| {
                Self::matches(r, urls, frame, loader)
                    && r.complete
                    && r.status.is_some()
                    && r.body.is_none()
                    && r.unavailable_reason.is_none()
            })
            .map(|(i, r)| (i, r.request_id.clone()))
            .collect()
    }

    pub fn store_body(&mut self, index: usize, result: Result<Value, String>) {
        let r = &mut self.records[index];
        if r.unavailable_reason.is_some() {
            return;
        }
        let decoded = result.and_then(|v| {
            let body = v["body"].as_str().ok_or("CDP returned no body")?;
            let encoded = v["base64Encoded"]
                .as_bool()
                .ok_or("CDP returned no body encoding")?;
            let remaining = MAX_BODY.min(MAX_TOTAL.saturating_sub(self.body_bytes));
            if body.len()
                > if encoded {
                    remaining.div_ceil(3) * 4
                } else {
                    remaining
                }
            {
                return Err("response body exceeds capture budget".into());
            }
            let bytes = if encoded {
                base64::engine::general_purpose::STANDARD
                    .decode(body)
                    .map_err(|_| "invalid CDP body encoding".to_string())?
            } else {
                body.as_bytes().to_vec()
            };
            if bytes.len() > remaining {
                return Err("response body exceeds capture budget".into());
            }
            Ok((bytes, !encoded))
        });
        match decoded {
            Ok((bytes, text)) => {
                self.body_bytes += bytes.len();
                r.body = Some(bytes);
                r.text = text;
            }
            Err(reason) => r.unavailable_reason = Some(reason),
        }
    }

    pub fn evidence(
        &self,
        urls: &[String],
        frame: &str,
        loader: &str,
        before: u64,
    ) -> ResponseEvidence {
        let mut responses: Vec<_> = self
            .records
            .iter()
            .filter(|r| Self::matches(r, urls, frame, loader))
            .cloned()
            .collect();
        let mut counts = HashMap::new();
        for r in &responses {
            *counts.entry(r.url.clone()).or_insert(0) += 1;
        }
        for r in &mut responses {
            r.ambiguous_url = counts[&r.url] > 1;
            if r.body.is_none() && r.unavailable_reason.is_none() {
                r.unavailable_reason = Some(
                    if !r.complete {
                        "response not complete"
                    } else {
                        "response metadata or body unavailable"
                    }
                    .into(),
                );
            }
            // An unavailable record must never also expose verified bytes.
            if r.unavailable_reason.is_some() {
                r.body = None;
            }
        }
        let missing_urls = urls
            .iter()
            .filter(|u| !counts.contains_key(*u))
            .cloned()
            .collect();
        ResponseEvidence {
            responses,
            missing_urls,
            truncated: self.truncated,
            changed_during_collection: before != self.revision,
            revision: self.revision,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;
    fn request(t: &mut ResponseCapture, id: &str, url: &str) {
        t.event(
            "Network.requestWillBeSent",
            &json!({"requestId":id,"request":{"url":url},"frameId":"main","loaderId":"current"}),
        );
    }
    #[test]
    fn redirects_cannot_borrow_final_body_and_repeated_urls_remain_ambiguous() {
        let mut t = ResponseCapture::default();
        request(&mut t, "1", "asset");
        request(&mut t, "1", "asset");
        t.event("Network.responseReceived",&json!({"requestId":"1","response":{"url":"asset","status":200,"mimeType":"image/png"}}));
        t.event("Network.loadingFinished", &json!({"requestId":"1"}));
        assert_eq!(
            t.pending(&["asset".into()], "main", "current"),
            vec![(1, "1".into())]
        );
        t.store_body(1, Ok(json!({"body":"AP8=","base64Encoded":true})));
        let e = t.evidence(&["asset".into()], "main", "current", t.revision);
        assert!(e.responses.iter().all(|r| r.ambiguous_url));
        assert!(e.responses[0].body.is_none());
        assert_eq!(e.responses[1].body, Some(vec![0, 255]));
    }
    #[test]
    fn browser_icon_fetch_is_neither_a_dependency_nor_a_change() {
        let mut t = ResponseCapture::default();
        let sent = |id: &str, url: &str, kind: &str, initiator: &str| json!({"requestId":id,"loaderId":"current","request":{"url":url},"frameId":"main","type":kind,"initiator":{"type":initiator}});
        t.event("Network.requestWillBeSent", &sent("current", "http://x/", "Document", "other"));
        let before = t.revision;
        // Chrome's tab-icon fetch, after load, on the main frame's loader.
        t.event("Network.requestWillBeSent", &sent("9.2", "http://x/favicon.ico", "Other", "other"));
        t.event("Network.responseReceived", &json!({"requestId":"9.2","response":{"url":"http://x/favicon.ico","status":404,"mimeType":"text/plain"}}));
        t.event("Network.loadingFinished", &json!({"requestId":"9.2"}));
        assert_eq!(t.revision, before, "the icon fetch must not read as the page changing");
        assert_eq!(t.urls("main", "current"), vec!["http://x/"]);
        // The document's own requests keep counting, even an `Other` one.
        t.event("Network.requestWillBeSent", &sent("9.3", "http://x/next.js", "Other", "parser"));
        assert_eq!(t.revision, before + 1);
        assert_eq!(t.urls("main", "current"), vec!["http://x/", "http://x/next.js"]);
    }
    #[test]
    fn incomplete_failed_and_wrong_document_are_explicit() {
        let mut t = ResponseCapture::default();
        request(&mut t, "1", "inflight");
        request(&mut t, "2", "failed");
        t.event(
            "Network.loadingFailed",
            &json!({"requestId":"2","errorText":"blocked"}),
        );
        let urls = vec!["inflight".into(), "failed".into(), "absent".into()];
        let e = t.evidence(&urls, "main", "current", 0);
        assert!(e.changed_during_collection);
        assert!(
            e.responses
                .iter()
                .all(|r| r.body.is_none() && r.unavailable_reason.is_some())
        );
        assert_eq!(e.missing_urls, vec!["absent"]);
        assert!(
            t.evidence(&urls, "iframe", "current", 0)
                .responses
                .is_empty()
        );
        assert!(t.evidence(&urls, "main", "old", 0).responses.is_empty());
        assert!(t.evidence(&urls, "main", "", 0).responses.is_empty());
    }
    #[test]
    fn decoded_text_follows_bom_then_lossy_utf8() {
        assert_eq!(decoded_text(b"\xEF\xBB\xBFa\xE9b"), "a\u{FFFD}b".as_bytes());
        assert_eq!(decoded_text(b"\xFF\xFEa\x00"), b"a");
        assert_eq!(decoded_text(b"\xFE\xFF\x00a\x00"), "a\u{FFFD}".as_bytes());
        assert_eq!(decoded_text(b"\xF0\x9F\x98x"), "\u{FFFD}x".as_bytes());
        let mut t = ResponseCapture::default();
        request(&mut t, "1", "a.css");
        t.store_body(0, Ok(json!({"body":"\u{FFFD}","base64Encoded":false})));
        let r = &t.records[0];
        assert!(r.text && r.body_matches(b"\xE9") && r.body_matches(b"\xEF\xBB\xBF\xE9"));
        assert!(!r.body_matches(b"\xC3\xA9") && !r.body_matches(b""));
        request(&mut t, "2", "b.bin");
        t.store_body(1, Ok(json!({"body":"/Q==","base64Encoded":true})));
        assert!(t.records[1].body_matches(b"\xFD") && !t.records[1].body_matches(b"\xEF\xBF\xBD"));
    }
    #[test]
    fn limits_and_invalid_encoding_never_become_empty_verified_bodies() {
        let mut t = ResponseCapture::default();
        request(&mut t, "1", "asset");
        t.store_body(0, Ok(json!({"body":"!","base64Encoded":true})));
        assert!(t.records[0].unavailable_reason.is_some());
        request(&mut t, "2", "large");
        t.body_bytes = MAX_TOTAL;
        t.store_body(1, Ok(json!({"body":"YQ==","base64Encoded":true})));
        assert!(t.records[1].unavailable_reason.is_some());
        for i in 2..=MAX_RECORDS {
            request(&mut t, &i.to_string(), "more");
        }
        assert!(t.truncated);
        assert_eq!(t.records.len(), MAX_RECORDS);
    }
}