Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
// SPDX-License-Identifier: FSL-1.1-MIT
// Copyright (c) 2026 Cua AI, Inc.
/**
* The page side of native video slots
* (apps/cua-spaces-macos/docs/native-video.md).
*
* On macOS the SwiftUI host draws live Space video itself, as native views
* over the web view. The page only says where, and whether the slot can be
* seen: each mounted slot (a Space tile, the Space viewer) reports its rect,
* the part of it its scroll containers leave visible (`clip`), whether
* page UI covers it (`occluded`: a dialog), and where floating UI sits over
* it (`holes`: a menu, a popover, a tooltip, a toast), all in CSS px
* relative to the web view's viewport, on the `cuaVideo` script message
* handler. The host registers that handler unless native video is turned
* off (`WebUINativeVideo` NO), so the handler's presence is the flag. No handler: the
* slot renders its fallback and nothing streams.
*
* Page → host (fire and forget):
*
* { type: "surfaces", update: SurfaceState[], remove: string[] }
* { type: "focus", surfaceId: string | null } // give keys to a viewer, or take them back
*
* Host → page (`cua:event`):
*
* { event: "video.surface", payload: { surfaceId, state: "connecting" | "live" | "failed", reason?, opening? } }
*
* `opening` (with `failed`): the host could not open a stream at all (its
* `streamProvider` threw), not one that failed once open.
* { event: "video.focus", payload: { surfaceId: string | null } }
*
* The host opens the Space's stream through its existing path
* (`LiveStreamSession(provider: backend.streamProvider(id:))`); the page
* never sees a ticket or a frame.
*/
export interface SurfaceRect {
x: number;
y: number;
width: number;
height: number;
}
/** A Space tile (low rate, small, view only) or the Space viewer (full rate, takes input). */
export type SurfaceTier = "tile" | "full";
export interface SurfaceState {
surfaceId: string;
spaceId: string;
tier: SurfaceTier;
/** Pointer, scroll and keys go to the Space (the viewer); a tile passes them to the page. */
interactive: boolean;
/** Where the video goes, inside the slot's border. */
rect: SurfaceRect;
/** The part of `rect` its scroll containers and the viewport leave visible; null when none is. */
clip: SurfaceRect | null;
radius: number;
/** A dialog (or anything else on top, but floating UI) covers part of the slot, or a modal dialog is open (any slot, the viewer too): the host hides the video. */
occluded: boolean;
/** Floating page UI over the slot (a menu, a popover, a tooltip, a toast, a tile's checkbox and power button while they show), in the same coordinates,
* inside `clip`: the host cuts the video out there and keeps the rest playing. */
holes?: SurfaceRect[];
/** On screen at all: the page is visible, the slot is laid out and not clipped away. */
visible: boolean;
}
export type StreamSurfaceMessage =
| { type: "surfaces"; update: SurfaceState[]; remove: string[] }
| { type: "focus"; surfaceId: string | null };
export type SurfacePhase = "connecting" | "live" | "failed";
export interface VideoHandler {
/** WebKit returns a promise that rejects when the host removed the handler. */
postMessage(message: StreamSurfaceMessage): unknown;
}
interface VideoHostWindow {
webkit?: { messageHandlers?: { cuaVideo?: { postMessage(message: unknown): unknown } } };
}
/** The host's `cuaVideo` handler, or null when the host draws no native video. */
export function nativeVideoHandler(win: unknown = globalThis.window): VideoHandler | null {
const handler = (win as VideoHostWindow | undefined)?.webkit?.messageHandlers?.cuaVideo;
return handler && typeof handler.postMessage === "function" ? (handler as VideoHandler) : null;
}