Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

10.7 KB
Raw Blame
// SPDX-License-Identifier: FSL-1.1-MIT
// Copyright (c) 2026 Cua AI, Inc.

//! The Space's screen, for the PiP view.
//!
//! Two ways in, both over rcdp wire v2:
//!
//! - [`open_ticket`]: mint a media session and hand the webview a ticketed
//!   `ws_url`; the page decodes H.264 with WebCodecs (the ticket is safe in a
//!   URL and authorizes that one session only).
//! - [`FrameWatch`] + [`watch_desktop`]: a native session whose frames are
//!   counted, not kept (memory-bounded by construction). The shell uses it
//!   for the stream health line; the scenario uses it to prove the first
//!   frame is a keyframe.

use crate::{Error, Result};
use cua_spaces::Space;
use cua_spaces::stream::{
    FrameSink, StreamEvent, StreamOptions, StreamSession, StreamStats, StreamTarget, StreamTicket,
    VideoFrame, WindowTarget,
};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;

/// Mints a desktop media session for the webview.
// #region docs:rs-stream
pub async fn open_ticket(space: &Space, max_fps: u32, max_dimension: u32) -> Result<StreamTicket> {
    Ok(space
        .open_stream(
            StreamTarget::Display(None),
            StreamOptions {
                max_fps,
                max_dimension,
                ..Default::default()
            },
        )
        .await?)
}
// #endregion docs:rs-stream

/// Mints a media session for one window (a per-window PiP).
pub async fn open_window_ticket(
    space: &Space,
    window_id: &str,
    max_fps: u32,
    max_dimension: u32,
) -> Result<StreamTicket> {
    if window_id.is_empty() {
        return Err(Error::Invalid("a window stream needs a window id".into()));
    }
    Ok(space
        .open_stream(
            StreamTarget::Window(window_id.into()),
            StreamOptions {
                max_fps,
                max_dimension,
                ..Default::default()
            },
        )
        .await?)
}

/// One row of the Computer panel's window list.
#[derive(Clone, Debug, PartialEq, serde::Serialize)]
pub struct WindowRow {
    pub window_id: String,
    pub app: String,
    pub title: String,
    pub width: f64,
    pub height: f64,
    /// The app's icon as the Space's desktop shows it, as a `data:` URL
    /// (from the SDK's `Space::app_icon`); absent when the Space has none,
    /// and then the row shows no icon at all.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub icon: Option<String>,
    #[serde(skip)]
    pub app_id: String,
    #[serde(skip)]
    pub pid: u32,
}

impl WindowRow {
    /// One row per window of the SDK's list (`Space::windows` lists only
    /// streamable windows).
    pub fn from_targets(targets: Vec<WindowTarget>) -> Vec<WindowRow> {
        targets
            .into_iter()
            .map(|w| WindowRow {
                window_id: w.window_id,
                app: w.app_name,
                title: w.title,
                width: w.bounds[2],
                height: w.bounds[3],
                icon: None,
                app_id: w.app_id,
                pid: w.pid,
            })
            .collect()
    }
}

/// `bytes` as a `data:` URL.
pub fn data_url(content_type: &str, bytes: &[u8]) -> String {
    const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
    let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4 + 32);
    out.push_str("data:");
    out.push_str(content_type);
    out.push_str(";base64,");
    for c in bytes.chunks(3) {
        let n = (c[0] as u32) << 16
            | (*c.get(1).unwrap_or(&0) as u32) << 8
            | *c.get(2).unwrap_or(&0) as u32;
        for (i, shift) in [18, 12, 6, 0].into_iter().enumerate() {
            if i <= c.len() {
                out.push(A[(n >> shift & 63) as usize] as char);
            } else {
                out.push('=');
            }
        }
    }
    out
}

/// The Space's streamable windows, each with its app icon when the Space
/// has one (every row's icon in one SDK call, through its one icon cache);
/// empty when it cannot stream single windows (no `window_stream`).
pub async fn windows(space: &Space) -> Result<Vec<WindowRow>> {
    if !space.supports("window_stream") {
        return Ok(Vec::new());
    }
    let mut rows = WindowRow::from_targets(space.windows(None).await?);
    let requests: Vec<cua_spaces::IconRequest> = rows
        .iter()
        .map(|r| cua_spaces::IconRequest {
            app_name: r.app.clone(),
            app_id: r.app_id.clone(),
            pid: r.pid,
        })
        .collect();
    if let Ok(icons) = space.app_icons(&requests).await {
        for (row, icon) in rows.iter_mut().zip(icons) {
            row.icon = icon.map(|i| data_url(i.content_type, &i.bytes));
        }
    }
    Ok(rows)
}

/// Counts frames and remembers what the first one was. Keeps no pixels.
#[derive(Default)]
pub struct FrameWatch {
    frames: AtomicU64,
    keyframes: AtomicU64,
    closed: AtomicU64,
    first: Mutex<Option<bool>>,
    size: Mutex<(u32, u32)>,
    codec: Mutex<String>,
}

impl FrameSink for FrameWatch {
    fn on_frame(&self, f: VideoFrame) {
        self.first.lock().unwrap().get_or_insert(f.keyframe);
        *self.size.lock().unwrap() = (f.width, f.height);
        {
            let mut c = self.codec.lock().unwrap();
            if c.is_empty() {
                *c = f.codec.clone();
            }
        }
        if f.keyframe {
            self.keyframes.fetch_add(1, Ordering::SeqCst);
        }
        self.frames.fetch_add(1, Ordering::SeqCst);
    }

    fn on_event(&self, e: StreamEvent) {
        if matches!(e, StreamEvent::Closed { .. }) {
            self.closed.fetch_add(1, Ordering::SeqCst);
        }
    }
}

/// A point-in-time view of a [`FrameWatch`].
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
pub struct WatchSnapshot {
    pub frames: u64,
    pub keyframes: u64,
    pub first_frame_keyframe: Option<bool>,
    pub width: u32,
    pub height: u32,
    pub codec: String,
    pub closed: bool,
}

impl FrameWatch {
    /// Current counters.
    pub fn snapshot(&self) -> WatchSnapshot {
        let (width, height) = *self.size.lock().unwrap();
        WatchSnapshot {
            frames: self.frames.load(Ordering::SeqCst),
            keyframes: self.keyframes.load(Ordering::SeqCst),
            first_frame_keyframe: *self.first.lock().unwrap(),
            width,
            height,
            codec: self.codec.lock().unwrap().clone(),
            closed: self.closed.load(Ordering::SeqCst) > 0,
        }
    }

    /// Waits until at least `n` frames arrived: at most `max_polls` polls of
    /// `poll` each, then fails.
    pub async fn wait_frames(&self, n: u64, poll: Duration, max_polls: u32) -> Result<()> {
        for _ in 0..max_polls {
            if self.frames.load(Ordering::SeqCst) >= n {
                return Ok(());
            }
            tokio::time::sleep(poll).await;
        }
        Err(Error::Timeout(format!(
            "{} of {n} frames after {max_polls} polls",
            self.frames.load(Ordering::SeqCst)
        )))
    }
}

/// A native desktop stream with its watch.
pub struct DesktopWatch {
    pub session: StreamSession,
    pub watch: Arc<FrameWatch>,
}

/// Opens a native desktop stream feeding a fresh [`FrameWatch`].
pub async fn watch_desktop(
    space: &Space,
    max_fps: u32,
    max_dimension: u32,
) -> Result<DesktopWatch> {
    let watch = Arc::new(FrameWatch::default());
    let session = space
        .stream_session(
            StreamTarget::Display(None),
            StreamOptions {
                max_fps,
                max_dimension,
                ..Default::default()
            },
            watch.clone(),
            None,
        )
        .await?;
    Ok(DesktopWatch { session, watch })
}

impl DesktopWatch {
    /// Asks for a keyframe (the decoder-resync path).
    pub fn request_keyframe(&self) -> Result<()> {
        Ok(self.session.request_keyframe()?)
    }

    /// Closes the session.
    pub async fn close(self) -> Result<StreamStats> {
        Ok(self.session.close().await?)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn frame(key: bool) -> VideoFrame {
        VideoFrame {
            sequence: 1,
            codec: "h264".into(),
            keyframe: key,
            width: 800,
            height: 600,
            capture_timestamp_us: 0,
            codec_epoch: 1,
            geometry_epoch: 1,
            data: vec![0; 4],
        }
    }

    #[test]
    fn the_watch_remembers_the_first_frame_and_keeps_no_pixels() {
        let w = FrameWatch::default();
        w.on_frame(frame(true));
        w.on_frame(frame(false));
        w.on_event(StreamEvent::Closed {
            code: 1000,
            reason: String::new(),
        });
        let s = w.snapshot();
        assert_eq!(s.frames, 2);
        assert_eq!(s.keyframes, 1);
        assert_eq!(s.first_frame_keyframe, Some(true));
        assert_eq!((s.width, s.height), (800, 600));
        assert_eq!(s.codec, "h264");
        assert!(s.closed);
    }

    fn target(id: &str) -> WindowTarget {
        WindowTarget {
            window_id: id.into(),
            epoch: 1,
            title: format!("{id} title"),
            app_name: "Thunar".into(),
            app_id: String::new(),
            pid: 7,
            bounds: [10.0, 20.0, 640.0, 480.0],
            on_screen: true,
            focused: false,
            available: true,
            limitation: String::new(),
        }
    }

    #[test]
    fn a_window_row_carries_the_title_and_size() {
        let rows = WindowRow::from_targets(vec![target("a")]);
        assert_eq!(
            rows,
            vec![WindowRow {
                window_id: "a".into(),
                app: "Thunar".into(),
                title: "a title".into(),
                width: 640.0,
                height: 480.0,
                icon: None,
                app_id: String::new(),
                pid: 7,
            }]
        );
        let json = serde_json::to_value(&rows[0]).unwrap();
        assert!(
            json.get("icon").is_none() && json.get("pid").is_none(),
            "{json}"
        );
    }

    #[test]
    fn icons_cross_as_data_urls() {
        assert_eq!(data_url("image/png", b""), "data:image/png;base64,");
        assert_eq!(data_url("image/png", b"f"), "data:image/png;base64,Zg==");
        assert_eq!(data_url("image/png", b"fo"), "data:image/png;base64,Zm8=");
        assert_eq!(
            data_url("image/svg+xml", b"foobar"),
            "data:image/svg+xml;base64,Zm9vYmFy"
        );
    }

    #[tokio::test]
    async fn waiting_for_frames_is_bounded() {
        let w = FrameWatch::default();
        let e = w
            .wait_frames(1, Duration::from_millis(1), 3)
            .await
            .unwrap_err();
        assert!(matches!(e, Error::Timeout(_)), "{e}");
    }
}