Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript
5.0 KB
Raw Blame
//! Screenshot encoding: ScreenCaptureKit hands back BGRA rows; the model gets a JPEG.

pub const JPEG_QUALITY: u8 = 65;

/// Encodes a BGRA bitmap whose rows are `bytes_per_row` long (rows may be padded) as JPEG.
pub fn bgra_to_jpeg(
    bgra: &[u8],
    width: u32,
    height: u32,
    bytes_per_row: usize,
    quality: u8,
) -> Result<Vec<u8>, String> {
    let (w, h) = (width as usize, height as usize);
    if w == 0 || h == 0 || bytes_per_row < w * 4 || bgra.len() < bytes_per_row * (h - 1) + w * 4 {
        let size = bgra.len();
        return Err(format!("unexpected bitmap: {width} × {height}, {bytes_per_row} bytes per row, {size} bytes"));
    }
    let mut rgb = Vec::with_capacity(w * h * 3);
    for row in bgra.chunks(bytes_per_row).take(h) {
        for i in (0..w * 4).step_by(4) {
            rgb.extend_from_slice(&[row[i + 2], row[i + 1], row[i]]);
        }
    }
    let mut jpeg = Vec::new();
    image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, quality)
        .encode(&rgb, width, height, image::ExtendedColorType::Rgb8)
        .map_err(|e| format!("cannot encode the screenshot: {e}"))?;
    Ok(jpeg)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn decode(jpeg: &[u8]) -> image::RgbImage {
        image::load_from_memory_with_format(jpeg, image::ImageFormat::Jpeg).unwrap().to_rgb8()
    }

    #[test]
    fn bgra_becomes_a_jpeg_of_the_same_size_and_colors() {
        // 8 × 8 pure red in BGRA.
        let bgra: Vec<u8> = [0u8, 0, 255, 255].repeat(64);
        let jpeg = bgra_to_jpeg(&bgra, 8, 8, 32, 90).unwrap();
        assert_eq!(&jpeg[..2], &[0xFF, 0xD8]);
        let image = decode(&jpeg);
        assert_eq!(image.dimensions(), (8, 8));
        let [r, g, b] = image.get_pixel(4, 4).0;
        assert!(r > 240 && g < 20 && b < 20, "{r} {g} {b}");
    }

    #[test]
    fn padded_rows_are_skipped() {
        // 8 × 8 blue, each row padded with 16 bytes of white.
        let row: Vec<u8> = [[255u8, 0, 0, 255].repeat(8), vec![255; 16]].concat();
        let jpeg = bgra_to_jpeg(&row.repeat(8), 8, 8, 48, 90).unwrap();
        let [r, g, b] = decode(&jpeg).get_pixel(7, 7).0;
        assert!(b > 240 && r < 20 && g < 20, "{r} {g} {b}");
    }

    /// A 1280 × 800 BGRA mock of an app window: light chrome, a sidebar, rows of 1 px
    /// "text" strokes, and two colored buttons.
    fn ui_like_bgra() -> Vec<u8> {
        let (w, h) = (1280usize, 800usize);
        let mut bgra = vec![0u8; w * h * 4];
        for y in 0..h {
            for x in 0..w {
                let (r, g, b) = if y < 52 {
                    (236, 236, 236)
                } else if x < 240 {
                    (246, 242, 248)
                } else if (600..640).contains(&y) && ((900..1010).contains(&x) || (1030..1140).contains(&x)) {
                    if x < 1010 { (10, 132, 255) } else { (255, 59, 48) }
                } else {
                    (255, 255, 255)
                };
                // Glyph-like strokes: 9 px tall lines of text every 24 px, broken into "words".
                let line = (y + 12) % 24 < 9 && y > 60 && y < 580;
                let glyph = (x * 7 + y * 3) % 11 < 4 && x % 64 < 52 && x > 20;
                let (r, g, b) = if line && glyph { (30, 30, 30) } else { (r, g, b) };
                let i = (y * w + x) * 4;
                bgra[i..i + 4].copy_from_slice(&[b, g, r, 255]);
            }
        }
        bgra
    }

    /// Mean absolute error per channel between the source and the decoded JPEG.
    fn mean_error(bgra: &[u8], jpeg: &[u8]) -> f64 {
        let decoded = decode(jpeg);
        let total: u64 = decoded
            .pixels()
            .zip(bgra.chunks(4))
            .map(|(p, s)| (0..3).map(|c| u64::from(p.0[c].abs_diff(s[2 - c]))).sum::<u64>())
            .sum();
        total as f64 / (decoded.pixels().len() * 3) as f64
    }

    #[test]
    fn the_screenshot_quality_is_lighter_and_keeps_text_legible() {
        // S24.8(c): quality 80 → JPEG_QUALITY. Sizes are printed for the record (`--nocapture`).
        assert!((60..=70).contains(&JPEG_QUALITY), "{JPEG_QUALITY}");
        let bgra = ui_like_bgra();
        let before = bgra_to_jpeg(&bgra, 1280, 800, 1280 * 4, 80).unwrap();
        let started = std::time::Instant::now();
        let after = bgra_to_jpeg(&bgra, 1280, 800, 1280 * 4, JPEG_QUALITY).unwrap();
        let took = started.elapsed().as_millis();
        let (error_before, error_after) = (mean_error(&bgra, &before), mean_error(&bgra, &after));
        eprintln!(
            "ui-like 1280x800: quality 80 = {} bytes (error {error_before:.2}), quality {JPEG_QUALITY} = {} bytes (error {error_after:.2}), encoded in {took} ms",
            before.len(),
            after.len()
        );
        assert!(after.len() * 100 <= before.len() * 85, "{} vs {}", after.len(), before.len());
        assert!(error_after < 6.0, "{error_after}");
    }

    #[test]
    fn short_or_empty_bitmaps_are_rejected() {
        assert!(bgra_to_jpeg(&[0; 10], 2, 2, 8, 80).is_err());
        assert!(bgra_to_jpeg(&[], 0, 0, 0, 80).is_err());
        assert!(bgra_to_jpeg(&[0; 64], 4, 4, 8, 80).is_err());
    }
}