Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
// The shared streaming scenario (examples/streaming/SCENARIO.md) in a page:
// @trycua/cua/browser (wasm, gRPC-Web) for the env calls, the media plane
// (MEDIA.md, rcdp wire v2) on a WebSocket, WebCodecs VideoDecoder -> canvas
// and AudioDecoder -> AudioContext / WAV.
//
// Interactive: fill in the form. Headless (headless.mjs, Playwright): the
// runner injects `window.__CUA_CONFIG` and `window.cuaReport(kind, data)`.
import { Cua, initialize } from "/sdk/index.js"
import { RGBA, fnv1a64, readTimecode, unixNs, wavBytes } from "./pixels.js"
const GRID_TITLE = "CUA Fixture Grid"
const GRID_LOG = "/tmp/cua-fixtures/grid.jsonl"
const CLICK_CELL = [2, 3]
const CLICK_CONTENT = { x: 200, y: 280 }
const CELL_RGB = [72, 153, 128]
const $ = (id) => document.getElementById(id)
const canvas = $("video")
const ctx = canvas.getContext("2d", { willReadFrequently: true })
function report(kind, data) {
if (typeof window.cuaReport === "function") window.cuaReport(kind, data)
if (kind === "log" || kind === "summary") {
const line = kind === "summary" ? `SUMMARY ${JSON.stringify(data)}` : data
$("log").textContent += line + "\n"
if (typeof window.cuaReport !== "function") console.log(line)
}
}
const log = (s) => report("log", s)
// ------------------------------------------------------------ bench JSONL
let benchOn = false
function jsonl(t, fields = {}) {
if (!benchOn) return
const rest = JSON.stringify(fields)
report("jsonl", `{"t":${JSON.stringify(t)},"unix_ns":${unixNs()}${rest === "{}" ? "" : "," + rest.slice(1, -1)}}`)
}
// ------------------------------------------------------------ targets
const num = (v) => (typeof v === "number" ? v : typeof v === "string" ? Number(v) : 0)
async function listTargets(env) {
const raw = JSON.parse(await env.callJson("StreamService/ListTargets", JSON.stringify({ includeWindows: true })))
const out = []
for (const t of raw.targets ?? []) {
const available = t.available ?? false
if (t.display) {
const d = t.display, b = d.bounds ?? {}, n = d.nativeSize ?? d.native_size ?? {}
out.push({ kind: "display", id: d.id ?? "", title: d.name ?? "", width: num(n.width) || num(b.width),
height: num(n.height) || num(b.height), x: num(b.x), y: num(b.y), available, primary: d.primary ?? false })
} else if (t.window) {
const w = t.window, b = w.bounds ?? {}
out.push({ kind: "window", id: w.ref?.id ?? "", title: w.title ?? "", width: num(b.width), height: num(b.height),
x: num(b.x), y: num(b.y), available, primary: false })
}
}
return out
}
const fmtTarget = (t) =>
`${t.kind.padEnd(7)} ${t.id.padEnd(24)} ${JSON.stringify(t.title).padEnd(28)} ${t.width}x${t.height} available=${t.available}`
// ------------------------------------------------------------ media plane
function findDescriptor(v, depth) {
if (!v || typeof v !== "object") return null
if ("sequence" in v && "codec" in v) return v
if (depth === 0) return null
for (const c of Object.values(v)) {
const d = findDescriptor(c, depth - 1)
if (d) return d
}
return null
}
/** `avc1.PPCCLL` from the first SPS of an Annex B access unit. */
function avcCodecString(au) {
for (let i = 0; i + 4 < au.length; i++) {
if (au[i] === 0 && au[i + 1] === 0 && (au[i + 2] === 1 || (au[i + 2] === 0 && au[i + 3] === 1))) {
const at = au[i + 2] === 1 ? i + 3 : i + 4
if ((au[at] & 0x1f) === 7 && at + 3 < au.length) {
const hex = (b) => b.toString(16).padStart(2, "0")
return `avc1.${hex(au[at + 1])}${hex(au[at + 2])}${hex(au[at + 3])}`
}
}
}
return null
}
class Stream {
constructor(env, cfg, target, { audio, wavName, playAudio, policy }) {
Object.assign(this, { env, cfg, target, audio, wavName, playAudio, policy })
this.frames = 0 // decoded frames
this.received = 0 // video packets
this.keyframes = 0
this.bytes = 0
this.audioPackets = 0
this.decodeErrors = 0
this.firstFrameAt = null
this.pending = new Map() // decoder timestamp -> packet meta
this.lastMeta = null
this.decoder = null
this.codecEpoch = null
this.needKey = true
this.tracks = new Map() // track_id -> { decoder, epoch, rate, channels, codec }
this.audioBytes = new Map() // pts_us -> encoded bytes
this.pcm = []
this.pcmRate = 48000
this.pcmChannels = 2
this.waiters = []
this.playhead = 0
}
async open() {
const t = this.target
const req = {
target: t.kind === "display" ? { displayId: t.id } : { window: { id: t.id } },
codecs: ["MEDIA_CODEC_H264"],
maxFps: 30,
...(this.audio ? { audio: { enabled: true } } : {}),
...(this.policy ? { policy: this.policy } : {}),
}
jsonl("open")
this.openedAt = performance.now()
const resp = JSON.parse(await this.env.callJson("StreamService/OpenMedia", JSON.stringify(req)))
this.sessionId = resp.mediaSessionId ?? resp.media_session_id
const ticket = resp.ticket
// The ticket goes in the subprotocol, not the URL (MEDIA.md §2).
const wsUrl = this.cfg.env.replace(/^http/, "ws").replace(/\/$/, "") + "/media"
this.ws = new WebSocket(wsUrl, ["rcdp.v2", `cua.ticket.${ticket}`])
this.ws.binaryType = "arraybuffer"
this.ws.onmessage = (m) => (typeof m.data === "string" ? this.onControl(m.data) : this.onBinary(new Uint8Array(m.data)))
this.closed = new Promise((ok) => (this.ws.onclose = (e) => ok(e)))
await new Promise((ok, fail) => {
this.ws.onopen = ok
this.ws.onerror = () => fail(new Error(`media socket ${wsUrl} failed`))
})
}
send(obj) {
if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(obj))
}
waitEvent(pred, ms) {
return new Promise((ok) => {
const waiter = (m) => {
if (!pred(m)) return false
clearTimeout(timer)
ok(m)
return true
}
const timer = setTimeout(() => {
this.waiters = this.waiters.filter((w) => w !== waiter)
ok(null)
}, ms)
this.waiters.push(waiter)
})
}
onControl(text) {
let m
try { m = JSON.parse(text) } catch { return }
if (m.type === "audio_config") this.configureAudio(m.payload)
if (m.type === "error") log(`media error ${JSON.stringify(m.payload)}`)
this.waiters = this.waiters.filter((w) => !w(m))
}
onBinary(b) {
if (b.length >= 24 && b[0] === 0x52 && b[1] === 0x41 && b[2] === 0x55 && b[3] === 0x32) return this.onAudio(b)
if (b.length < 8) return
const v = new DataView(b.buffer, b.byteOffset, b.byteLength)
const hl = v.getUint32(0), pl = v.getUint32(4)
if (hl > 1 << 20 || 8 + hl + pl !== b.length) return
let d
try { d = findDescriptor(JSON.parse(new TextDecoder().decode(b.subarray(8, 8 + hl))), 4) } catch { return }
if (!d) return
this.onVideo(d, b.subarray(8 + hl))
}
requestKeyframe() {
const now = performance.now()
if (this.lastKeyReq && now - this.lastKeyReq < 1000) return
this.lastKeyReq = now
this.send({ type: "request_keyframe", payload: { session_id: this.sessionId } })
}
onVideo(d, payload) {
this.received++
if (d.keyframe) this.keyframes++
this.bytes += payload.length
const meta = {
seq: d.sequence, bytes: payload.length, key: !!d.keyframe, cap_us: d.capture_timestamp_us,
w: d.width_px, h: d.height_px, geometry_epoch: d.geometry_epoch,
}
if (d.codec !== "h264") return // bgra/png are loopback fallbacks; not handled here
if (this.decoder && this.codecEpoch !== d.codec_epoch) {
this.decoder.close()
this.decoder = null
this.needKey = true
}
if (this.needKey && !d.keyframe) return this.requestKeyframe()
if (!this.decoder) {
const codec = avcCodecString(payload)
if (!codec) return this.requestKeyframe()
this.decoder = new VideoDecoder({
output: (f) => this.onDecoded(f),
error: (e) => {
this.decodeErrors++
log(`decode error ${e.message}`)
this.decoder = null
this.needKey = true
},
})
// No `description`: the bitstream is Annex B with in-band SPS/PPS.
this.decoder.configure({ codec, optimizeForLatency: true, hardwareAcceleration: "no-preference" })
this.codecEpoch = d.codec_epoch
}
this.needKey = false
// Timestamps must be unique per chunk: use the capture time.
const ts = Number(d.capture_timestamp_us)
this.pending.set(ts, meta)
if (this.pending.size > 128) this.pending.delete(this.pending.keys().next().value)
try {
this.decoder.decode(new EncodedVideoChunk({ type: d.keyframe ? "key" : "delta", timestamp: ts, data: payload }))
} catch (e) {
this.decodeErrors++
this.needKey = true
this.requestKeyframe()
}
}
onDecoded(frame) {
const clientMs = performance.timeOrigin + performance.now()
const meta = this.pending.get(frame.timestamp) ?? {}
this.pending.delete(frame.timestamp)
this.frames++
if (this.firstFrameAt === null) this.firstFrameAt = performance.now()
const w = frame.displayWidth, h = frame.displayHeight
if (canvas.width !== w || canvas.height !== h) {
canvas.width = w
canvas.height = h
}
ctx.drawImage(frame, 0, 0)
frame.close()
this.lastMeta = meta
if (benchOn) {
const scale = w / (this.target.width || w)
const stripH = Math.min(h, Math.ceil(16 * scale))
const strip = ctx.getImageData(0, 0, w, stripH).data
const tc = readTimecode(strip, w, stripH, w * 4, scale, RGBA, clientMs)
jsonl("frame", { seq: meta.seq ?? null, bytes: meta.bytes ?? null, key: meta.key ?? null,
cap_us: meta.cap_us ?? frame.timestamp, w, h, tc_ms: tc })
}
}
configureAudio(p) {
if (p.direction && p.direction !== "down") return
const prev = this.tracks.get(p.track_id)
prev?.decoder?.close?.()
const track = { epoch: p.config_epoch, rate: p.sample_rate_hz, channels: p.channels, codec: p.codec, decoder: null }
if (p.codec === "opus") {
track.decoder = new AudioDecoder({
output: (a) => this.onPcm(track, a),
error: (e) => {
this.decodeErrors++
log(`audio decode error ${e.message}`)
},
})
track.decoder.configure({ codec: "opus", sampleRate: p.sample_rate_hz, numberOfChannels: p.channels })
}
this.tracks.set(p.track_id, track)
}
onAudio(b) {
const v = new DataView(b.buffer, b.byteOffset, b.byteLength)
if (b[4] !== 2 || b[23] !== 0 || (b[5] & 0xfc) !== 0) return
const trackId = v.getUint16(6)
const track = this.tracks.get(trackId)
if (!track || b[22] !== track.epoch) return
const pts = Number(v.getBigUint64(12))
const frameSamples = v.getUint16(20)
const payload = b.subarray(24)
this.audioPackets++
if (track.codec === "opus" && track.decoder?.state === "configured") {
this.audioBytes.set(pts, payload.length)
if (this.audioBytes.size > 256) this.audioBytes.delete(this.audioBytes.keys().next().value)
track.decoder.decode(new EncodedAudioChunk({ type: "key", timestamp: pts, data: payload }))
} else if (track.codec === "pcm_s16le") {
const s = new Int16Array(payload.length / 2)
for (let i = 0; i < s.length; i++) s[i] = v.getInt16(24 + i * 2, true)
this.pushPcm(s, track.rate, track.channels, pts, payload.length, frameSamples)
}
}
onPcm(track, a) {
const n = a.numberOfFrames, ch = a.numberOfChannels
const planes = []
for (let c = 0; c < ch; c++) {
const plane = new Float32Array(n)
a.copyTo(plane, { planeIndex: c, format: "f32-planar" })
planes.push(plane)
}
const s = new Int16Array(n * ch)
for (let i = 0; i < n; i++)
for (let c = 0; c < ch; c++) s[i * ch + c] = Math.max(-32768, Math.min(32767, Math.round(planes[c][i] * 32767)))
const pts = a.timestamp
const bytes = this.audioBytes.get(pts) ?? null
this.audioBytes.delete(pts)
a.close()
this.pushPcm(s, a.sampleRate ?? track.rate, ch, pts, bytes, n, planes)
}
pushPcm(s, rate, ch, pts, bytes, samples, planes) {
this.pcmRate = rate
this.pcmChannels = ch
if (this.wavName) this.pcm.push(s)
jsonl("audio", { pts_us: pts, bytes, samples })
if (this.playAudio) this.play(s, rate, ch, planes)
}
play(s, rate, ch, planes) {
if (!this.ac) {
this.ac = new AudioContext({ sampleRate: rate, latencyHint: "interactive" })
this.ac.resume().catch(() => {})
}
const n = s.length / ch
const buf = this.ac.createBuffer(ch, n, rate)
for (let c = 0; c < ch; c++) {
const out = buf.getChannelData(c)
if (planes) out.set(planes[c])
else for (let i = 0; i < n; i++) out[i] = s[i * ch + c] / 32768
}
const src = this.ac.createBufferSource()
src.buffer = buf
src.connect(this.ac.destination)
// Small jitter buffer (40 ms, MEDIA.md §12.5) without drift correction.
const now = this.ac.currentTime
if (this.playhead < now + 0.02 || this.playhead > now + 0.25) this.playhead = now + 0.04
src.start(this.playhead)
this.playhead += n / rate
}
async close() {
const endAt = performance.now()
this.ws.close(1000)
await Promise.race([this.closed, new Promise((ok) => setTimeout(ok, 2000))])
try { await this.env.callJson("StreamService/CloseMedia", JSON.stringify({ mediaSessionId: this.sessionId })) } catch {}
const flush = (d) => (d && d.state === "configured" ? Promise.race([d.flush().catch(() => {}), new Promise((ok) => setTimeout(ok, 1000))]) : null)
await Promise.all([flush(this.decoder), ...[...this.tracks.values()].map((t) => flush(t.decoder))])
this.decoder?.state !== "closed" && this.decoder?.close()
for (const t of this.tracks.values()) t.decoder?.state !== "closed" && t.decoder?.close()
await this.ac?.close()
return endAt
}
lastBgraHash() {
if (!this.frames) return null
const rgba = ctx.getImageData(0, 0, canvas.width, canvas.height).data
const bgra = new Uint8Array(rgba.length)
for (let i = 0; i < rgba.length; i += 4) {
bgra[i] = rgba[i + 2]; bgra[i + 1] = rgba[i + 1]; bgra[i + 2] = rgba[i]; bgra[i + 3] = rgba[i + 3]
}
return fnv1a64(bgra)
}
}
async function runStream(env, cfg, target, opts, during) {
const s = new Stream(env, cfg, target, opts)
await s.open()
const deadline = s.openedAt + cfg.seconds * 1000
if (during) {
for (let i = 0; i < 100 && s.frames === 0 && performance.now() < deadline; i++) await sleep(50)
await during(s)
}
const left = deadline - performance.now()
if (left > 0) await sleep(left)
const endAt = await s.close()
const first = s.firstFrameAt === null ? null : s.firstFrameAt - s.openedAt
const span = s.firstFrameAt === null ? 0 : (endAt - s.firstFrameAt) / 1000
let wav = null
if (opts.wavName) {
const bytes = wavBytes(s.pcm, s.pcmRate, s.pcmChannels)
wav = `${cfg.outDir}/${opts.wavName}`
report("file", { name: opts.wavName, base64: toBase64(bytes) })
if (typeof window.cuaReport !== "function") {
const a = document.createElement("a")
a.href = URL.createObjectURL(new Blob([bytes], { type: "audio/wav" }))
a.download = opts.wavName
a.textContent = ` ${opts.wavName}`
$("downloads").append(a)
}
}
const stats = {
frames: s.frames,
received: s.received,
keyframes: s.keyframes,
bytes: s.bytes,
first_frame_ms: first === null ? null : Math.round(first * 10) / 10,
fps: span > 0 ? Math.round((s.frames / span) * 10) / 10 : 0,
audio_packets: s.audioPackets,
last_hash: s.lastBgraHash(),
hash_of: "bgra (canvas readback)",
wav,
decode_errors: s.decodeErrors,
}
return [stats, s]
}
const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
function toBase64(bytes) {
let s = ""
for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000))
return btoa(s)
}
// ------------------------------------------------------------ click
async function gridLogLines(env) {
const out = await env.sh(`cat ${GRID_LOG} 2>/dev/null || true`, 10000)
return new TextDecoder().decode(out.stdout).split("\n").filter(Boolean)
}
function isCellPress(line) {
try {
const v = JSON.parse(line)
const kind = v.event ?? v.kind ?? v.type
return kind === "button_press" && Array.isArray(v.cell) && v.cell[0] === CLICK_CELL[0] && v.cell[1] === CLICK_CELL[1]
} catch {
return false
}
}
function clickPoint(meta, win) {
const w = canvas.width, h = canvas.height
return [(CLICK_CONTENT.x * w) / (win.width || w), (CLICK_CONTENT.y * h) / (win.height || h)]
}
async function clickCell(env, s, win) {
const result = { sent: false, via: null, logged: false, pixel_ok: null }
const before = (await gridLogLines(env)).length
const meta = s.lastMeta
if (meta) {
const [x, y] = clickPoint(meta, win)
const actionId = `web-click-${Date.now()}`
s.send({ type: "action", payload: { action_id: actionId, session_id: s.sessionId, tool: "click",
arguments: { x: Math.round(x), y: Math.round(y) },
basis: { kind: "pixel", geometry_epoch: meta.geometry_epoch, frame_sequence: meta.seq } } })
result.sent = true
result.via = "action"
const ack = await s.waitEvent((m) => m.type === "action_result" && m.payload?.action_id === actionId, 3000)
result.action_delivered = ack ? ack.payload.delivered === true : null
if (!ack || ack.payload.delivered !== true)
result.action_error = ack ? JSON.stringify(ack.payload.error ?? ack.payload) : "no action_result within 3 s"
}
if (!result.sent || result.action_error) {
await env.click(win.x + CLICK_CONTENT.x, win.y + CLICK_CONTENT.y)
result.sent = true
result.via = "env"
}
for (let i = 0; i < 20 && !result.logged; i++) {
await sleep(250)
result.logged = (await gridLogLines(env)).slice(before).some(isCellPress)
}
return result
}
// ------------------------------------------------------------ main
async function run(cfg) {
await initialize()
const cua = Cua.embedded()
const env = await cua.spacesd(cfg.env, cfg.token)
if (cfg.bench) {
benchOn = true
const targets = await listTargets(env)
const [kind, ...rest] = cfg.benchTarget.split(":")
const key = rest.join(":")
const target = kind === "window"
? targets.find((t) => t.kind === "window" && t.title === key)
: key === "primary"
? targets.find((t) => t.kind === "display" && t.primary) ?? targets.find((t) => t.kind === "display")
: targets.find((t) => t.kind === "display" && t.id === key)
if (!target) {
log(`bench target ${cfg.benchTarget} not found; targets:\n${targets.map(fmtTarget).join("\n")}`)
return 1
}
const [stats] = await runStream(env, cfg, target, { audio: cfg.benchAudio, wavName: null, playAudio: false })
log(`BENCH ${JSON.stringify({ example: "typescript-web", target: cfg.benchTarget, ...stats })}`)
return stats.frames > 0 ? 0 : 1
}
log(`health ${await env.health()}`)
const fx = await env.sh("cua-fixtures start grid", 30000)
log(`fixture grid: exit=${fx.exit.code ?? fx.exit.signal ?? fx.exit.error} ${new TextDecoder().decode(fx.stdout).trim()}`)
let targets = [], win
for (let i = 0; i < 40 && !win; i++) {
targets = await listTargets(env)
win = targets.find((t) => t.kind === "window" && t.title === GRID_TITLE)
if (!win) await sleep(250)
}
for (const t of targets) log(`target ${fmtTarget(t)}`)
const display = targets.find((t) => t.kind === "display" && t.primary) ?? targets.find((t) => t.kind === "display")
if (!display) throw new Error("no display target")
if (!win) throw new Error(`no window titled ${GRID_TITLE}`)
const play = !cfg.headless
const [desktop] = await runStream(env, cfg, display, { audio: true, wavName: "desktop.wav", playAudio: play })
log(`desktop ${JSON.stringify(desktop)}`)
let click = { sent: false, via: null, logged: false, pixel_ok: null }
const [window_, ws] = await runStream(
env, cfg, win,
{ audio: true, wavName: "window.wav", playAudio: play, policy: "SESSION_POLICY_ALLOW_ACTIVATION" },
async (s) => {
await sleep(Math.min(1000, (cfg.seconds * 1000) / 4))
click = await clickCell(env, s, win)
},
)
if (ws.frames) {
const [x, y] = clickPoint(ws.lastMeta, win)
const px = [...ctx.getImageData(Math.round(x), Math.round(y), 1, 1).data.subarray(0, 3)]
click.pixel = px
click.pixel_ok = px.every((c, i) => Math.abs(c - CELL_RGB[i]) <= 24)
}
log(`window ${JSON.stringify(window_)}`)
report("summary", { example: "typescript-web", desktop, window: window_, click })
return desktop.frames > 0 && window_.frames > 0 && click.logged ? 0 : 1
}
function start(cfg) {
run(cfg).then(
(code) => report("done", code),
(e) => {
log(`error: ${e?.stack ?? e}`)
report("done", 1)
},
)
}
if (window.__CUA_CONFIG) {
document.getElementById("form").hidden = true
start({ seconds: 5, outDir: "./out", headless: true, bench: false, benchAudio: true, ...window.__CUA_CONFIG })
} else {
const q = new URLSearchParams(location.search)
if (q.get("env")) $("env").value = q.get("env")
if (q.get("seconds")) $("seconds").value = q.get("seconds")
$("form").addEventListener("submit", (e) => {
e.preventDefault()
$("run").disabled = true
$("log").textContent = ""
start({ env: $("env").value.trim(), token: $("token").value, seconds: Number($("seconds").value) || 5,
outDir: "(download)", headless: false, bench: false })
})
}