Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

18.8 KB
Raw Blame
//! Streaming example on the cua SDK (Rust host of the `cua-sdk` crate).
//!
//! Runs the shared scenario in `examples/streaming/SCENARIO.md`: connect,
//! start the grid fixture, list targets, stream the desktop and the grid
//! window (decoded BGRA + PCM), click a grid cell over the media socket,
//! verify it, and print a `SUMMARY` line. With `CUA_BENCH_JSONL` set it runs
//! the benchmark lane instead (one target, per-frame JSONL).
//!
//! Headless only: rendering lives in `cua-viewer` (libs/cua/crates/cua-viewer),
//! which uses the same media plane.

use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use cua_sdk::{
    Cua, CuaConfig, DecodedFrameSink, DecodedVideoFrame, SpacesdClient, MediaEvent, MediaOpenOptions,
    MediaSession, PcmAudio, PcmSink,
};
use serde_json::{json, Value};

type Error = Box<dyn std::error::Error + Send + Sync>;

const GRID_TITLE: &str = "CUA Fixture Grid";
const CLICK_CONTENT_PX: (f64, f64) = (200.0, 280.0);
const CELL_RGB: [i32; 3] = [72, 153, 128];
/// Hard bound on WAV size per stream (memory safety): 60 s of 48 kHz stereo.
const MAX_PCM_SAMPLES: usize = 48_000 * 2 * 60;

fn unix_ns() -> i64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|d| d.as_nanos() as i64)
        .unwrap_or(0)
}

fn cpu_s() -> (f64, f64) {
    // SAFETY: getrusage fills the zeroed struct we own.
    unsafe {
        let mut u: libc::rusage = std::mem::zeroed();
        libc::getrusage(libc::RUSAGE_SELF, &mut u);
        let tv = |t: libc::timeval| t.tv_sec as f64 + t.tv_usec as f64 / 1e6;
        (tv(u.ru_utime), tv(u.ru_stime))
    }
}

// ------------------------------------------------------------------ pixels

fn fnv1a(data: &[u8]) -> u64 {
    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
    for b in data {
        h ^= u64::from(*b);
        h = h.wrapping_mul(0x0100_0000_01b3);
    }
    h
}

fn rgb_at(frame: &DecodedVideoFrame, x: u32, y: u32) -> Option<[i32; 3]> {
    if x >= frame.width || y >= frame.height {
        return None;
    }
    let i = (y * frame.stride + x * 4) as usize;
    let p = frame.data.get(i..i + 4)?;
    Some([i32::from(p[2]), i32::from(p[1]), i32::from(p[0])])
}

fn cell_bit(frame: &DecodedVideoFrame, ox: u32, oy: u32, index: u32) -> Option<u32> {
    let mut sum = 0i32;
    for dy in 6..10 {
        for dx in 6..10 {
            let rgb = rgb_at(frame, ox + index * 16 + dx, oy + dy)?;
            sum += (rgb[0] + rgb[1] + rgb[2]) / 3;
        }
    }
    Some(u32::from(sum / 16 > 128))
}

/// The bench timecode (SCENARIO.md "Bench timecode") at a known origin.
fn read_timecode_at(frame: &DecodedVideoFrame, ox: u32, oy: u32) -> Option<u32> {
    for (i, want) in [1, 0, 1, 0].iter().enumerate() {
        if cell_bit(frame, ox, oy, i as u32)? != *want {
            return None;
        }
    }
    for (i, want) in [0, 1, 0, 1].iter().enumerate() {
        if cell_bit(frame, ox, oy, 44 + i as u32)? != *want {
            return None;
        }
    }
    let mut value = 0u32;
    for i in 0..32 {
        value = (value << 1) | cell_bit(frame, ox, oy, 4 + i)?;
    }
    let mut check = 0u32;
    for i in 0..8 {
        check = (check << 1) | cell_bit(frame, ox, oy, 36 + i)?;
    }
    let b = value.to_be_bytes();
    (u32::from(b[0] ^ b[1] ^ b[2] ^ b[3]) == check).then_some(value)
}

/// Window frames: the strip is at (0, 0); otherwise scan (bounded).
fn read_timecode(frame: &DecodedVideoFrame, origin: &mut Option<(u32, u32)>) -> Option<u32> {
    if let Some((x, y)) = *origin {
        if let Some(v) = read_timecode_at(frame, x, y) {
            return Some(v);
        }
    }
    if frame.width < 768 || frame.height < 16 {
        return None;
    }
    for y in (0..=frame.height - 16).step_by(2).take(2000) {
        for x in (0..=frame.width - 768).step_by(2) {
            if cell_bit(frame, x, y, 0) == Some(1) && cell_bit(frame, x, y, 1) == Some(0) {
                if let Some(v) = read_timecode_at(frame, x, y) {
                    *origin = Some((x, y));
                    return Some(v);
                }
            }
        }
    }
    None
}

fn unwrap_ms(low: u32, now_ms: i64) -> i64 {
    let base = now_ms & !0xFFFF_FFFF;
    let mut best = base | i64::from(low);
    for c in [best - (1 << 32), best + (1 << 32)] {
        if (c - now_ms).abs() < (best - now_ms).abs() {
            best = c;
        }
    }
    best
}

fn write_wav(path: &Path, samples: &[i16], rate: u32, channels: u16) -> std::io::Result<()> {
    let mut f = std::io::BufWriter::new(std::fs::File::create(path)?);
    let data_len = (samples.len() * 2) as u32;
    f.write_all(b"RIFF")?;
    f.write_all(&(36 + data_len).to_le_bytes())?;
    f.write_all(b"WAVEfmt ")?;
    f.write_all(&16u32.to_le_bytes())?;
    f.write_all(&1u16.to_le_bytes())?;
    f.write_all(&channels.to_le_bytes())?;
    f.write_all(&rate.to_le_bytes())?;
    f.write_all(&(rate * u32::from(channels) * 2).to_le_bytes())?;
    f.write_all(&(channels * 2).to_le_bytes())?;
    f.write_all(&16u16.to_le_bytes())?;
    f.write_all(b"data")?;
    f.write_all(&data_len.to_le_bytes())?;
    for s in samples {
        f.write_all(&s.to_le_bytes())?;
    }
    f.flush()
}

// ------------------------------------------------------------------- sinks

#[derive(Default)]
struct State {
    frames: u64,
    first_frame_at: Option<Instant>,
    last: Option<DecodedVideoFrame>,
    last_hash: u64,
    geometry_epoch: u64,
    audio_packets: u64,
    pcm: Vec<i16>,
    rate: u32,
    channels: u16,
    events: Vec<String>,
    input_acks: u64,
    tc_origin: Option<(u32, u32)>,
    jsonl: Option<std::fs::File>,
}

struct Sink(Mutex<State>);

impl Sink {
    fn new(jsonl: Option<std::fs::File>) -> Arc<Self> {
        Arc::new(Self(Mutex::new(State {
            jsonl,
            ..State::default()
        })))
    }
}

impl DecodedFrameSink for Sink {
    fn on_decoded_frame(&self, frame: DecodedVideoFrame) {
        let now = unix_ns();
        let mut s = self.0.lock().unwrap();
        s.frames += 1;
        s.first_frame_at.get_or_insert_with(Instant::now);
        if s.jsonl.is_some() {
            let mut origin = s.tc_origin;
            let tc = read_timecode(&frame, &mut origin);
            s.tc_origin = origin;
            let line = json!({"t":"frame","unix_ns":now,"seq":frame.sequence,"bytes":null,"key":null,
                "cap_us":frame.capture_timestamp_us,"w":frame.width,"h":frame.height,
                "tc_ms": tc.map(|v| unwrap_ms(v, now / 1_000_000))});
            if let Some(f) = s.jsonl.as_mut() {
                let _ = writeln!(f, "{line}");
            }
        }
        s.geometry_epoch = frame.geometry_epoch;
        s.last = Some(frame);
    }

    fn on_event(&self, event: MediaEvent) {
        let mut s = self.0.lock().unwrap();
        if event.kind == "interactive_input_acknowledgement" {
            s.input_acks += 1;
        }
        if s.events.len() < 256 {
            s.events.push(event.kind.clone());
        }
        if event.kind == "session_opened" {
            if let Some(e) = serde_json::from_str::<Value>(&event.json)
                .ok()
                .and_then(|v| v.pointer("/payload/geometry_epoch").and_then(Value::as_u64))
            {
                s.geometry_epoch = e;
            }
        }
    }
}

impl PcmSink for Sink {
    fn on_pcm(&self, audio: PcmAudio) {
        let now = unix_ns();
        let mut s = self.0.lock().unwrap();
        s.audio_packets += 1;
        s.rate = audio.sample_rate;
        s.channels = audio.channels;
        if s.pcm.len() + audio.samples.len() <= MAX_PCM_SAMPLES {
            s.pcm.extend_from_slice(&audio.samples);
        }
        if s.jsonl.is_some() {
            let line = json!({"t":"audio","unix_ns":now,"pts_us":audio.pts_us,"bytes":null,
                "samples": audio.samples.len() / usize::from(audio.channels.max(1))});
            if let Some(f) = s.jsonl.as_mut() {
                let _ = writeln!(f, "{line}");
            }
        }
    }
}

// --------------------------------------------------------------- scenario

struct Target {
    kind: &'static str,
    id: String,
    title: String,
    width: f64,
    height: f64,
}

async fn list_targets(env: &SpacesdClient) -> Result<Vec<Target>, Error> {
    let raw = env
        .call_json(
            "/cua.env.v1.StreamService/ListTargets".into(),
            json!({"includeWindows": true}).to_string(),
            None,
        )
        .await?;
    let v: Value = serde_json::from_str(&raw)?;
    let mut out = Vec::new();
    for t in v
        .get("targets")
        .and_then(Value::as_array)
        .into_iter()
        .flatten()
    {
        if let Some(d) = t.get("display") {
            let size = d.get("nativeSize").cloned().unwrap_or_default();
            out.push(Target {
                kind: "display",
                id: d
                    .get("id")
                    .and_then(Value::as_str)
                    .unwrap_or_default()
                    .into(),
                title: d
                    .get("name")
                    .and_then(Value::as_str)
                    .unwrap_or_default()
                    .into(),
                width: size.get("width").and_then(Value::as_f64).unwrap_or(0.0),
                height: size.get("height").and_then(Value::as_f64).unwrap_or(0.0),
            });
        } else if let Some(w) = t.get("window") {
            let r = w.get("ref").cloned().unwrap_or_default();
            let b = w.get("bounds").cloned().unwrap_or_default();
            let num = |v: &Value, k: &str| v.get(k).and_then(Value::as_f64).unwrap_or(0.0);
            out.push(Target {
                kind: "window",
                id: r
                    .get("id")
                    .and_then(Value::as_str)
                    .unwrap_or_default()
                    .into(),
                title: w
                    .get("title")
                    .and_then(Value::as_str)
                    .unwrap_or_default()
                    .into(),
                width: num(&b, "width"),
                height: num(&b, "height"),
            });
        }
    }
    Ok(out)
}

fn options(target: &Target, audio: bool) -> MediaOpenOptions {
    let window = target.kind == "window";
    // Input needs a policy other than view-only; the escape hatch sets it.
    let request =
        json!({"policy": "SESSION_POLICY_ALLOW_ACTIVATION", "codecs": ["MEDIA_CODEC_H264"]});
    MediaOpenOptions {
        display: (!window).then(|| target.id.clone()),
        window_handle: window.then(|| target.id.clone()),
        max_fps: 30,
        max_dimension: 0,
        audio,
        disable_video: false,
        request_json: Some(request.to_string()),
    }
}

struct StreamReport {
    summary: Value,
    session: Arc<MediaSession>,
    sink: Arc<Sink>,
}

async fn stream(
    env: &SpacesdClient,
    target: &Target,
    seconds: f64,
    out: &Path,
    name: &str,
    jsonl: Option<std::fs::File>,
) -> Result<StreamReport, Error> {
    let sink = Sink::new(jsonl);
    let opened = Instant::now();
    let session = env
        .open_media_decoded_with_audio(options(target, true), sink.clone(), sink.clone())
        .await?;
    tokio::time::sleep(Duration::from_secs_f64(seconds)).await;
    let s = sink.0.lock().unwrap();
    let wav = out.join(format!("{name}.wav"));
    if s.rate > 0 {
        write_wav(&wav, &s.pcm, s.rate, s.channels)?;
    }
    let last_hash = s
        .last
        .as_ref()
        .map(|f| fnv1a(&f.data))
        .unwrap_or(s.last_hash);
    let summary = json!({
        "frames": s.frames,
        "keyframes": null,
        "bytes": null,
        "first_frame_ms": s.first_frame_at.map(|t| (t.duration_since(opened).as_secs_f64() * 1e4).round() / 10.0),
        "fps": ((s.frames as f64 / seconds) * 10.0).round() / 10.0,
        "audio_packets": s.audio_packets,
        "last_hash": format!("{last_hash:016x}"),
        "hash_of": "bgra",
        "size": s.last.as_ref().map(|f| [f.width, f.height]),
        "wav": if s.rate > 0 { Value::from(wav.display().to_string()) } else { Value::Null },
    });
    drop(s);
    Ok(StreamReport {
        summary,
        session,
        sink,
    })
}

async fn grid_log(env: &SpacesdClient) -> Vec<Value> {
    match env
        .sh(
            "cat /tmp/cua-fixtures/grid.jsonl 2>/dev/null || true".into(),
            Some(10_000),
        )
        .await
    {
        Ok(out) => String::from_utf8_lossy(&out.stdout)
            .lines()
            .filter_map(|l| serde_json::from_str(l).ok())
            .collect(),
        Err(_) => Vec::new(),
    }
}

/// Click grid cell (2, 3) through the media socket (interactive pointer
/// input), then check the fixture log and the decoded pixel.
async fn click(env: &SpacesdClient, report: &StreamReport, window: &Target) -> Value {
    let before = grid_log(env).await.len();
    let (fw, fh) = {
        let s = report.sink.0.lock().unwrap();
        s.last
            .as_ref()
            .map(|f| (f64::from(f.width), f64::from(f.height)))
            .unwrap_or((window.width, window.height))
    };
    let scale = if window.width > 0.0 {
        fw / window.width
    } else {
        1.0
    };
    let (x, y) = (CLICK_CONTENT_PX.0 * scale, CLICK_CONTENT_PX.1 * scale);
    let pointer = |phase: &str| {
        json!({"kind":"pointer","phase":phase,"button":"left",
        "x_normalized": x / fw, "y_normalized": y / fh, "modifiers": []})
    };
    let batch = json!({"type":"interactive_input","payload":{
        "session_id": report.session.session_id(), "first_sequence": 1,
        "events": [pointer("move"), pointer("down"), pointer("up")]}});
    let sent = report.session.send_control(batch.to_string()).is_ok();
    let mut logged = false;
    for _ in 0..20 {
        tokio::time::sleep(Duration::from_millis(250)).await;
        let lines = grid_log(env).await;
        if lines.iter().skip(before).any(|l| {
            l.get("type").and_then(Value::as_str) == Some("button_press")
                && l.get("cell") == Some(&json!([2, 3]))
        }) {
            logged = true;
            break;
        }
    }
    let pixel = {
        let s = report.sink.0.lock().unwrap();
        s.last.as_ref().and_then(|f| rgb_at(f, x as u32, y as u32))
    };
    let pixel_ok = pixel.is_some_and(|p| p.iter().zip(CELL_RGB).all(|(a, b)| (a - b).abs() <= 24));
    json!({"sent": sent, "via": "interactive_input", "logged": logged, "pixel_ok": pixel_ok,
        "frame_point": [x.round(), y.round()], "pixel": pixel})
}

async fn scenario(env: &SpacesdClient) -> Result<bool, Error> {
    let seconds: f64 = std::env::var("CUA_STREAM_SECONDS")
        .ok()
        .and_then(|s| s.parse().ok())
        .unwrap_or(5.0);
    let out = PathBuf::from(std::env::var("CUA_OUT_DIR").unwrap_or_else(|_| "./out".into()));
    std::fs::create_dir_all(&out)?;
    println!("health {}", env.health().await?);
    let started = env
        .sh("cua-fixtures start grid".into(), Some(30_000))
        .await?;
    println!("{}", String::from_utf8_lossy(&started.stdout).trim());

    let mut targets = Vec::new();
    for _ in 0..20 {
        targets = list_targets(env).await?;
        if targets.iter().any(|t| t.title == GRID_TITLE) {
            break;
        }
        tokio::time::sleep(Duration::from_millis(250)).await;
    }
    for t in &targets {
        println!(
            "target {:7} {:24} {:32} {}x{}",
            t.kind, t.id, t.title, t.width, t.height
        );
    }
    let display = targets
        .iter()
        .find(|t| t.kind == "display")
        .ok_or("no display target")?;
    let grid = targets
        .iter()
        .find(|t| t.title == GRID_TITLE)
        .ok_or("grid window not listed")?;

    let desktop = stream(env, display, seconds, &out, "desktop", None).await?;
    desktop.session.close().await?;
    let window = stream(env, grid, seconds, &out, "window", None).await?;
    let click = click(env, &window, grid).await;
    window.session.close().await?;
    let ok = desktop.summary["frames"].as_u64().unwrap_or(0) > 0
        && window.summary["frames"].as_u64().unwrap_or(0) > 0
        && click["logged"] == json!(true);
    let summary = json!({"example": "rust", "desktop": desktop.summary, "window": window.summary, "click": click});
    println!("SUMMARY {summary}");
    Ok(ok)
}

async fn bench(env: &SpacesdClient, path: &str) -> Result<bool, Error> {
    let seconds: f64 = std::env::var("CUA_BENCH_SECONDS")
        .ok()
        .and_then(|s| s.parse().ok())
        .unwrap_or(10.0);
    let spec = std::env::var("CUA_BENCH_TARGET").unwrap_or_else(|_| "display:primary".into());
    let audio = std::env::var("CUA_BENCH_AUDIO")
        .map(|v| v != "0")
        .unwrap_or(true);
    let targets = list_targets(env).await?;
    let target = match spec.split_once(':') {
        Some(("window", title)) => targets
            .into_iter()
            .find(|t| t.kind == "window" && t.title == title),
        Some(("display", id)) => targets
            .into_iter()
            .find(|t| t.kind == "display" && (id == "primary" || t.id == id)),
        _ => None,
    }
    .ok_or_else(|| format!("bench target {spec} not found"))?;
    let mut file = std::fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(path)?;
    writeln!(file, "{}", json!({"t": "open", "unix_ns": unix_ns()}))?;
    let sink = Sink::new(Some(file));
    let mut opts = options(&target, audio);
    opts.audio = audio;
    let session = env
        .open_media_decoded_with_audio(opts, sink.clone(), sink.clone())
        .await?;
    tokio::time::sleep(Duration::from_secs_f64(seconds)).await;
    session.close().await?;
    let (user, sys) = cpu_s();
    let mut s = sink.0.lock().unwrap();
    let frames = s.frames;
    if let Some(f) = s.jsonl.as_mut() {
        writeln!(
            f,
            "{}",
            json!({"t": "end", "unix_ns": unix_ns(), "cpu_user_s": user, "cpu_sys_s": sys})
        )?;
    }
    eprintln!("bench: {frames} frames");
    Ok(frames > 0)
}

#[tokio::main]
async fn main() -> std::process::ExitCode {
    let url = std::env::var("CUA_ENV_URL").unwrap_or_else(|_| "http://127.0.0.1:33211".into());
    let Ok(token) = std::env::var("CUA_ENV_TOKEN") else {
        eprintln!("CUA_ENV_TOKEN is required (see examples/streaming/SCENARIO.md)");
        return std::process::ExitCode::from(2);
    };
    let result = async {
        let cua = Cua::embedded(CuaConfig::default())?;
        let env = cua.spacesd(url, Some(token)).await?;
        let run = async {
            match std::env::var("CUA_BENCH_JSONL") {
                Ok(path) => bench(&env, &path).await,
                Err(_) => scenario(&env).await,
            }
        };
        tokio::time::timeout(Duration::from_secs(300), run)
            .await
            .map_err(|_| Error::from("scenario timed out"))?
    }
    .await;
    match result {
        Ok(true) => std::process::ExitCode::SUCCESS,
        Ok(false) => std::process::ExitCode::from(1),
        Err(error) => {
            eprintln!("error: {error}");
            std::process::ExitCode::from(1)
        }
    }
}