Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
/**
* OpenKoalaBots web UI: a Bots roster, one long-lived thread per Bot, and
* the Bot's computer beside it. Talks to the local server (`npm run serve`)
* for Spaces, Bots, files and presence, and streams the Space's
* desktop itself: a media ticket (from the server, or opened directly over
* gRPC-Web with the wasm `@trycua/cua/browser` client) attaches the `/media`
* WebSocket, and WebCodecs decodes H.264 into a canvas.
*/
import { MediaSocketState, avcCodecString, encodeControl, parseBinary } from "../src/core/wire.js"
import { attachMedia, closeMedia, openDesktopMedia, type JsonRpc, type OpenedMedia } from "../src/core/webmedia.js"
import { awaitingApproval, rowPreview, statusLabel, toBlocks, type Block, type FileEvent, type ThreadItem } from "../src/ui/blocks.js"
import { renderActivity } from "../src/ui/activity.js"
import { append, h, icon } from "../src/ui/dom.js"
import { friendlyError } from "../src/ui/errors.js"
import { useImageList } from "../src/ui/images.js"
import { renderDropZone, renderWindowList, type WindowRow } from "../src/ui/computer.js"
import { renderSpaceSection } from "../src/ui/spaceSection.js"
import {
botAvatar,
setAssignedColors,
useBotAvatarImage,
renderDrawables,
renderGroupThread,
renderGroupsSection,
renderNewGroupSheet,
renderRoutinesPanel,
renderRoutinesSection,
type BotRef,
type CursorView,
type GroupView,
type RoutineView,
} from "../src/ui/coworkers.js"
import { APP_TELEPORT_MESSAGE } from "../src/core/teleport.js"
import {
classifyBrowserDrop,
defineTeleportElements,
readDataTransfer,
idleDropZone,
sendingLabel,
type DropZoneFile,
type DropZoneStatus,
} from "@trycua/cua/teleport"
import { PictureInPicture, desktopSource, pipKey, pipLabel, pipSize, windowSource } from "@trycua/cua/spaces/pip"
import { PresenceView, type PresenceEvent, type PresenceMember } from "@trycua/cua/spaces/presence"
import { renderWizard } from "../src/ui/wizardView.js"
import type { SpacePlan, WizardState } from "../src/ui/wizard.js"
import imageList from "../../../libs/images/sandbox-images.json"
import koalaMark from "./assets/koala-mark.svg"
import koalaPeek from "./assets/koala-peek.svg"
import koalaSleep from "./assets/koala-sleep.svg"
// The shared image list, imported (one source for the docs and every picker).
useImageList(imageList)
const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T
const params = new URLSearchParams(location.hash.slice(1))
const token = params.get("token") ?? ""
async function api<T = unknown>(path: string, init: { method?: string; body?: unknown; raw?: Blob; query?: string } = {}): Promise<T> {
const res = await fetch(`/api/${path}${init.query ?? ""}`, {
method: init.method ?? (init.body !== undefined || init.raw ? "POST" : "GET"),
headers: { authorization: `Bearer ${token}`, ...(init.raw ? {} : { "content-type": "application/json" }) },
body: init.raw ?? (init.body === undefined ? undefined : JSON.stringify(init.body)),
})
const value = (await res.json()) as T & { error?: string }
if (!res.ok) throw new Error(value.error ?? `${res.status}`)
return value
}
// ------------------------------------------------------------------ state
interface BotView {
runId: string
bot: { id: string; name: string; agent: string }
state: string
reason: string
acceptsMessage: boolean
/** The thread as the SDK's event fold made it. */
items: ThreadItem[]
/** The Bot's last message on one line. */
preview: string
}
interface SpaceView {
id: string
name: string
provider: string
spacesdVersion?: string
features: string[]
}
interface Snapshot {
space: SpaceView | null
bots: BotView[]
knownBots?: BotRef[]
routines?: RoutineView[]
groups?: GroupView[]
presence?: { me: string | null; cursors: CursorView[]; colors?: Record<string, string> } | null
}
const AGENTS = [
{ id: "claude-code", label: "Claude Code" },
{ id: "openai-codex", label: "Codex" },
]
const SETTLED = new Set(["idle", "finished", "failed", "crashed", "stopped", "awaiting_input"])
const state = {
connected: false,
cloud: false,
spaces: [] as SpaceView[],
selectedSpace: null as string | null,
snap: { space: null, bots: [] } as Snapshot,
/** A Bot hired in the page that has not had its first task yet. */
pending: null as { name: string; agent: string } | null,
selected: null as string | null, // runId, or "pending"
/** The main pane: a Bot's thread, a Bot's routines, or a group chat. */
view: { kind: "bot" } as { kind: "bot" } | { kind: "routines"; botID: string } | { kind: "group"; id: string },
files: {} as Record<string, FileEvent[]>,
query: "",
computerOpen: true,
error: "",
notice: "",
stream: { on: true, frames: 0, width: 0, height: 0, status: "" },
}
/** Activity groups the user expanded (`runId:block:item`), kept across renders. */
const openActivity = new Set<string>()
function fail(e: unknown): void {
state.error = e instanceof Error ? e.message : String(e)
renderMain()
}
// ------------------------------------------------------------- the frame
const app = $("app")
const sidebar = h("aside", { class: "sidebar" })
const main = h("main", { class: "thread" })
const computer = h("aside", { class: "computer" })
app.append(sidebar, main, computer)
useBotAvatarImage(koalaMark)
function avatar(size = ""): HTMLElement {
return h("span", { class: `avatar ${size}` }, h("img", { src: koalaMark, alt: "" }))
}
// --------------------------------------------------------------- sidebar
function knownBots(): BotRef[] {
return state.snap.knownBots ?? []
}
function openView(v: typeof state.view): void {
state.view = v
if (v.kind !== "bot") state.selected = null
render()
}
function renderSidebar(): void {
const roster = h("nav", { class: "roster" })
renderRosterInto(roster)
const search = h("input", {
placeholder: "Search Bots",
value: state.query,
oninput: (e) => {
state.query = (e.target as HTMLInputElement).value
roster.replaceChildren()
renderRosterInto(roster)
},
})
sidebar.replaceChildren(
h("div", { class: "brand" }, h("img", { src: koalaMark, alt: "" }), h("b", {}, "OpenKoalaBots")),
h(
"div",
{ class: "sidebar-top" },
h("button", { class: "btn new-bot", onclick: () => (state.selectedSpace ? openNewBot() : openWizard()) }, icon("plus"), "New Bot"),
h("label", { class: "search" }, icon("search"), search),
),
h("div", { class: "section-label" }, "Bots"),
roster,
...(state.snap.space
? [
renderRoutinesSection({
routines: state.snap.routines ?? [],
bots: knownBots(),
selectedBot: state.view.kind === "routines" ? state.view.botID : null,
onOpen: (botID) => openView({ kind: "routines", botID }),
}),
renderGroupsSection({
groups: state.snap.groups ?? [],
selected: state.view.kind === "group" ? state.view.id : null,
botCount: knownBots().length,
onOpen: (id) => openView({ kind: "group", id }),
onNew: () => openNewGroup(),
}),
]
: []),
renderSpaceSection({
spaces: state.spaces,
selected: state.selectedSpace,
onSelect: (id) => selectSpace(id),
onNewSpace: () => openWizard(),
onDelete: (id) =>
api("spaces/delete", { body: { id } })
.then(() => refreshSpaces())
.catch(fail),
}),
h(
"div",
{ class: "account" },
h("span", { class: "avatar sm initials", style: "background:#5d5d63" }, "OP"),
h("span", { class: "who" }, h("div", { class: "name" }, "Operator"), h("div", { class: "sub" }, state.connected ? (state.cloud ? "Cua Cloud connected" : "This machine only") : "Connecting…")),
h("span", { class: `dot ${state.connected ? (state.cloud ? "ready" : "none") : "unknown"}` }),
),
)
}
function renderRosterInto(el: HTMLElement): void {
const q = state.query.toLowerCase()
let shown = 0
if (state.pending && state.pending.name.toLowerCase().includes(q)) {
el.append(botRow("pending", state.pending.name, "No messages yet", "none", false))
shown++
}
for (const b of state.snap.bots) {
if (!b.bot.name.toLowerCase().includes(q)) continue
el.append(botRow(b.runId, b.bot.name, rowPreview(b.preview, b.items) || "No messages yet", b.state, awaitingApproval(toBlocks(b.items)), b.bot.id))
shown++
}
const total = state.snap.bots.length + (state.pending ? 1 : 0)
if (total === 0) el.append(h("div", { class: "roster-empty" }, "No Bots yet. Hire one to get started."))
else if (shown === 0) el.append(h("div", { class: "roster-empty" }, `No Bot matches “${state.query}”.`))
}
function botRow(id: string, name: string, last: string, status: string, needsYou: boolean, botID?: string): HTMLElement {
return h(
"button",
{
class: `bot-row ${state.selected === id ? "on" : ""}`,
onclick: () => {
state.selected = id
state.view = { kind: "bot" }
render()
},
},
botID ? botAvatar(botID) : avatar(),
h("span", { class: "who" }, h("span", { class: "name" }, name), h("span", { class: "last" }, last)),
h("span", { class: "meta" }, needsYou ? h("span", { class: "badge" }, "1") : h("span", { class: `dot ${status}`, title: statusLabel(status) })),
)
}
// ------------------------------------------------------------------ main
const composers = new Map<string, HTMLElement>()
/** One composer per Bot (kept across renders, so a draft survives). */
function composer(key: string, opts: { big?: boolean; bare?: boolean; placeholder: string; agent: string; agentLocked?: boolean; onAgent?: (a: string) => void; onSend: (t: string) => void; onAttach?: (f: File) => void; lead?: HTMLElement }): HTMLElement {
const k = `${key}:${opts.big ? "big" : "dock"}`
const cached = composers.get(k)
if (cached) return cached
const area = h("textarea", { rows: opts.big ? 2 : 1, placeholder: opts.placeholder })
const send = h("button", { class: "send", title: "Send", disabled: true }, icon("arrowUp", 18))
const submit = () => {
const t = area.value.trim()
if (!t || send.disabled) return
area.value = ""
send.disabled = true
opts.onSend(t)
}
area.addEventListener("input", () => (send.disabled = !area.value.trim()))
area.addEventListener("keydown", (e) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault()
submit()
}
})
send.addEventListener("click", submit)
const file = h("input", {
type: "file",
hidden: true,
onchange: (e) => {
const f = (e.target as HTMLInputElement).files?.[0]
if (f) opts.onAttach?.(f)
;(e.target as HTMLInputElement).value = ""
},
})
const picker = h(
"select",
{ class: "agent-picker", title: opts.agentLocked ? "The thread keeps its agent" : "Agent", disabled: !!opts.agentLocked, onchange: (e) => opts.onAgent?.((e.target as HTMLSelectElement).value) },
...AGENTS.map((a) => h("option", { value: a.id }, a.label)),
)
picker.value = opts.agent
const el = h(
"div",
{ class: `composer ${opts.big ? "big" : ""}` },
opts.lead ?? null,
area,
h(
"div",
{ class: "tools" },
opts.bare ? null : h("button", { class: "icon-btn", title: "Attach a file", disabled: !opts.onAttach, onclick: () => file.click() }, icon("paperclip")),
opts.bare ? null : file,
opts.bare ? null : picker,
h("span", { class: "spacer" }),
send,
),
)
composers.set(k, el)
return el
}
function empty(art: string, title: string, text: string, ...rest: Array<HTMLElement | null>): HTMLElement {
return h("div", { class: "empty" }, h("img", { class: "art", src: art, alt: "" }), h("h1", {}, title), h("p", {}, text), ...rest)
}
function banner(kind: "error" | "", text: string, clear: () => void): HTMLElement {
const f = kind === "error" ? friendlyError(text) : { message: text, detail: "" }
return h(
"div",
{ class: `banner ${kind}` },
icon(kind === "error" ? "alert" : "check"),
h(
"span",
{ class: "selectable" },
f.message,
f.link ? h("a", { class: "btn btn-primary", href: f.link.url, target: "_blank", rel: "noopener noreferrer" }, f.link.label) : null,
f.detail ? h("details", {}, h("summary", {}, "Details"), f.detail) : null,
),
h(
"button",
{
class: "icon-btn",
title: "Dismiss",
onclick: () => {
clear()
renderMain()
},
},
icon("close"),
),
)
}
let lastScrollKey = ""
function renderMain(): void {
if (state.view.kind !== "bot" && state.snap.space) return renderSideView(state.view)
const bot = state.snap.bots.find((b) => b.runId === state.selected) ?? null
const pending = state.selected === "pending" ? state.pending : null
const header = h("header", { class: "pane-header" })
if (bot || pending) {
const status = bot?.state ?? ""
append(header, [
bot ? botAvatar(bot.bot.id, "sm") : avatar("sm"),
h("span", { class: "title" }, bot?.bot.name ?? pending!.name),
h("span", { class: `dot ${status || "none"}` }),
h("span", { class: "subtitle" }, `${statusLabel(status)} · ${AGENTS.find((a) => a.id === (bot?.bot.agent ?? pending!.agent))?.label ?? ""}`),
bot ? h("button", { class: "icon-btn", title: "Routines", onclick: () => openView({ kind: "routines", botID: bot.bot.id }) }, icon("clock")) : null,
])
} else header.append(h("span", { class: "title" }, state.snap.space?.name ?? "OpenKoalaBots"))
header.append(
h("span", { class: "spacer" }),
h(
"button",
{
class: `icon-btn ${state.computerOpen ? "on" : ""}`,
title: state.computerOpen ? "Hide the Computer" : "Show the Computer",
onclick: () => {
state.computerOpen = !state.computerOpen
app.classList.toggle("computer-closed", !state.computerOpen)
renderMain()
},
},
icon("panelRight"),
),
)
const parts: HTMLElement[] = [header]
if (state.error) parts.push(banner("error", state.error, () => (state.error = "")))
if (state.notice) parts.push(banner("", state.notice, () => (state.notice = "")))
if (!state.snap.space) {
parts.push(
empty(
koalaPeek,
"Give your Bots a computer",
"Bots work in a Space: a Linux, Windows or macOS desktop in the cloud or on this machine.",
h(
"div",
{ class: "suggestions" },
h("button", { class: "btn btn-primary", onclick: () => openWizard() }, icon("plus"), "New Space"),
h("button", { class: "btn", onclick: () => openAddress() }, icon("link"), "Add by address"),
),
),
)
} else if (pending) {
parts.push(
empty(
koalaSleep,
`What should ${pending.name} work on?`,
"Give it a task. It keeps this thread, so follow-ups pick up where it left off.",
composer(`pending:${pending.name}`, { big: true, placeholder: `Message ${pending.name}…`, agent: pending.agent, onAgent: (a) => (pending.agent = a), onSend: (t) => hire(pending.name, pending.agent, t) }),
),
)
} else if (!bot) {
const name = h("input", { value: "Koala", "aria-label": "Bot name", style: "border:0;background:transparent;outline:none;width:120px;font-weight:600;color:var(--text)" })
let agent = AGENTS[0].id
parts.push(
empty(
koalaPeek,
"Hire a Bot to get started",
"Name it and give it a first task. It works on its own computer, and you can watch or step in.",
composer("hire", {
big: true,
placeholder: "Describe the first task…",
agent,
onAgent: (a) => (agent = a),
onSend: (t) => hire((name as HTMLInputElement).value.trim() || "Koala", agent, t),
lead: h("label", { class: "attach-chip", style: "margin-left:-6px" }, "Name", name),
}),
),
)
} else {
const blocks = toBlocks(bot.items, state.files[bot.runId] ?? [])
const col = h("div", { class: "col" }, ...blocks.map((b, i) => blockView(b, bot, i === blocks.length - 1, i)))
if (!SETTLED.has(bot.state))
col.append(h("div", { class: "msg msg-bot" }, botAvatar(bot.bot.id), h("div", { class: "body" }, h("div", { class: "card status" }, h("span", { class: "spinner" }), h("b", {}, bot.bot.name), " is working in its Space…"))))
const scroll = h("div", { class: "scroll" }, col)
parts.push(
scroll,
h(
"div",
{ class: "dock" },
bot.acceptsMessage ? null : h("div", { class: "note" }, icon("info"), `${bot.bot.name} is mid-turn: a message now would be refused, not queued.`),
composer(bot.runId, {
placeholder: `Message ${bot.bot.name}…`,
agent: bot.bot.agent,
agentLocked: true,
onSend: (t) => message(bot.runId, t),
onAttach: (f) => attachFile(f, bot.runId),
}),
),
)
const key = `${bot.runId}:${JSON.stringify(bot.items.at(-1) ?? null)}:${bot.items.length}:${bot.state}`
if (key !== lastScrollKey) {
lastScrollKey = key
requestAnimationFrame(() => (scroll.scrollTop = scroll.scrollHeight))
}
}
const prev = main.querySelector(".scroll")
const keep = prev ? prev.scrollTop : 0
main.replaceChildren(...parts)
const now = main.querySelector(".scroll")
if (now && prev) now.scrollTop = keep
}
/** The Routines panel of a Bot, or a group chat, in the main pane. */
function renderSideView(v: { kind: "routines"; botID: string } | { kind: "group"; id: string }): void {
const header = h("header", { class: "pane-header" })
const parts: HTMLElement[] = [header]
const names = Object.fromEntries(knownBots().map((b) => [b.id, b.name]))
if (v.kind === "routines") {
const bot = knownBots().find((b) => b.id === v.botID) ?? { id: v.botID, name: v.botID }
const thread = state.snap.bots.filter((b) => b.bot.id === bot.id).at(-1)
header.append(
icon("clock"),
h("span", { class: "title" }, `Routines · ${bot.name}`),
h("span", { class: "spacer" }),
)
if (thread) header.append(h("button", { class: "btn btn-ghost", onclick: () => ((state.selected = thread.runId), openView({ kind: "bot" })) }, "Open thread"))
if (state.error) parts.push(banner("error", state.error, () => (state.error = "")))
parts.push(
h(
"div",
{ class: "scroll" },
h(
"div",
{ class: "col" },
renderRoutinesPanel({
bot,
routines: (state.snap.routines ?? []).filter((r) => r.botID === bot.id),
onCreate: (r) => void api("routines", { body: { botID: bot.id, ...r } }).then(refreshState, fail),
onToggle: (id, enabled) => void api(`routines/${encodeURIComponent(id)}`, { body: { enabled } }).then(refreshState, fail),
onRun: (id) => void api(`routines/${encodeURIComponent(id)}/run`, { body: {} }).then(refreshState, fail),
onDelete: (id) => void api(`routines/${encodeURIComponent(id)}/delete`, { body: {} }).then(refreshState, fail),
}),
),
),
)
} else {
const g = (state.snap.groups ?? []).find((x) => x.id === v.id)
if (!g) {
state.view = { kind: "bot" }
return renderMain()
}
header.append(h("span", { class: "title" }, g.title), h("span", { class: "subtitle" }, g.membershipLabel))
if (state.error) parts.push(banner("error", state.error, () => (state.error = "")))
const scroll = renderGroupThread({ group: g, names })
parts.push(
scroll,
h(
"div",
{ class: "dock" },
composer(`group:${g.id}`, {
bare: true,
placeholder: `Message ${g.memberIDs.map((id) => names[id] ?? id).join(", ")}…`,
agent: "",
onSend: (text) => void api(`groups/${encodeURIComponent(g.id)}/send`, { body: { text } }).then(refreshState, fail),
}),
),
)
const key = `${g.id}:${g.messages.length}:${g.working.length}`
if (key !== lastScrollKey) {
lastScrollKey = key
requestAnimationFrame(() => (scroll.scrollTop = scroll.scrollHeight))
}
}
main.replaceChildren(...parts)
}
async function refreshState(): Promise<void> {
state.snap = await api<Snapshot>("state")
render()
}
function openNewGroup(): void {
const el = renderNewGroupSheet({
bots: knownBots(),
onCancel: () => close(),
onCreate: (title, members) => {
api<GroupView>("groups", { body: { title, members } })
.then(async (g) => {
close()
await refreshState()
openView({ kind: "group", id: g.id })
})
.catch((e) => {
close()
fail(e)
})
},
})
const close = sheet(el)
}
function blockView(block: Block, bot: BotView, live: boolean, index: number): HTMLElement {
if (block.kind === "user") return h("div", { class: "msg msg-user" }, h("div", { class: "bubble" }, block.text))
const items = block.items.map((it, i) => {
switch (it.kind) {
case "text":
return h("div", { class: "text" }, it.text)
case "activity": {
const key = `${bot.runId}:${index}:${i}`
return renderActivity({
summary: it.summary,
steps: it.steps,
open: openActivity.has(key),
onToggle: (open) => (open ? openActivity.add(key) : openActivity.delete(key)),
})
}
case "approval":
return h(
"div",
{ class: "card approval" },
h("div", { class: "card-head" }, icon("alert"), `${bot.bot.name} needs your approval`),
h("div", { class: "card-body" }, it.prompt),
live && i === block.items.length - 1
? h(
"div",
{ class: "actions" },
h("button", { class: "btn btn-primary", onclick: () => message(bot.runId, it.options[0]) }, "Approve"),
h("button", { class: "btn", onclick: () => message(bot.runId, it.options[1]) }, "Deny"),
)
: null,
)
case "file":
return h("div", { class: "card file" }, h("span", { class: "file-icon" }, icon("file")), h("span", {}, h("div", { class: "name" }, it.name), h("div", { class: "sub" }, it.detail)))
}
})
return h("div", { class: "msg msg-bot" }, botAvatar(bot.bot.id), h("div", { class: "body" }, h("div", { class: "author" }, bot.bot.name), ...items))
}
// ------------------------------------------------------------------ actions
async function hire(name: string, agent: string, prompt: string): Promise<void> {
try {
const r = await api<{ runId: string }>("bots", { body: { name, agent, prompt } })
state.pending = null
state.selected = r.runId
state.error = ""
render()
} catch (e) {
fail(e)
}
}
async function message(runId: string, text: string): Promise<void> {
try {
const r = await api<{ accepted: boolean; reason: string }>(`bots/${encodeURIComponent(runId)}/message`, { body: { text } })
if (!r.accepted) state.error = `Refused: ${r.reason}`
renderMain()
} catch (e) {
fail(e)
}
}
async function attachFile(file: File, runId: string | null, quiet = false): Promise<{ bytes: number; dest: string } | null> {
try {
const r = await api<{ files: Array<{ path: string; size: number }>; bytes: number; verified: boolean }>("files", { raw: file, query: `?name=${encodeURIComponent(file.name)}` })
const dest = r.files[0]?.path ?? "~/Downloads/openkoalabots"
const detail = `${formatBytes(Number(r.bytes))} to ${dest}${r.verified ? ", SHA-256 verified" : ""}`
if (runId) {
const bot = state.snap.bots.find((b) => b.runId === runId)
const turn = bot?.items.at(-1)?.turn ?? 0
;(state.files[runId] ??= []).push({ turn, name: file.name, detail })
} else if (!quiet) state.notice = `Sent ${file.name}: ${detail}`
renderMain()
return r.verified ? { bytes: Number(r.bytes), dest } : null
} catch (e) {
fail(e)
return null
}
}
function formatBytes(n: number): string {
if (n < 1024) return `${n} B`
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
return `${(n / (1024 * 1024)).toFixed(1)} MB`
}
async function selectSpace(id: string): Promise<void> {
try {
await api("spaces/select", { body: { id } })
state.selectedSpace = id
state.selected = null
state.pending = null
await refreshSpaces()
} catch (e) {
fail(e)
}
}
async function refreshSpaces(): Promise<void> {
const r = await api<{ spaces: SpaceView[]; selected: string | null }>("spaces")
state.spaces = r.spaces
state.selectedSpace = r.selected
render()
}
// ----------------------------------------------------------------- sheets
function sheet(el: HTMLElement): () => void {
document.body.append(el)
return () => el.remove()
}
function openWizard(initial?: Partial<WizardState>): void {
let close = () => {}
const el = renderWizard({
cloud: state.cloud,
initial,
onCancel: () => close(),
onAddByAddress: () => {
close()
openAddress()
},
onCreate: async (plan: SpacePlan, openWhenReady: boolean) => {
try {
const info = await api<SpaceView>("spaces/create", { body: { plan } })
state.notice = `${info.name} is ready.`
if (openWhenReady) {
state.computerOpen = true
app.classList.remove("computer-closed")
}
await refreshSpaces()
return true
} catch (e) {
fail(e)
return false
}
},
})
close = sheet(el)
}
function field(label: string, input: HTMLElement): HTMLElement {
return h("label", { class: "field" }, h("span", {}, label), input)
}
function openAddress(): void {
const url = h("input", { class: "input", placeholder: "10.0.0.5:3211", autofocus: true })
const tok = h("input", { class: "input", type: "password", placeholder: "The spacesd token" })
const name = h("input", { class: "input", placeholder: "Optional" })
const add = h("button", { class: "btn btn-primary" }, "Add Space")
const el = h(
"div",
{ class: "scrim", role: "dialog", "aria-modal": "true" },
h(
"div",
{ class: "sheet sm" },
h("div", { class: "sheet-head" }, h("h2", {}, "Add a Space by address"), h("p", {}, "A machine that already runs cua-spacesd. Nothing is created.")),
h("div", { class: "sheet-body" }, field("Address", url), field("Token", tok), field("Name", name)),
h("div", { class: "sheet-foot" }, h("span", { class: "spacer" }), h("button", { class: "btn", onclick: () => close() }, "Cancel"), add),
),
)
add.onclick = async () => {
add.disabled = true
try {
await api("spaces/add", { body: { url: (url as HTMLInputElement).value.trim(), token: (tok as HTMLInputElement).value, name: (name as HTMLInputElement).value.trim() } })
close()
await refreshSpaces()
} catch (e) {
close()
fail(e)
}
}
const close = sheet(el)
}
function openNewBot(): void {
const name = h("input", { class: "input", placeholder: "Research, Inbox, Koala…", autofocus: true })
const agent = h("select", { class: "select" }, ...AGENTS.map((a) => h("option", { value: a.id }, a.label)))
const go = () => {
const n = (name as HTMLInputElement).value.trim()
if (!n) return
state.pending = { name: n, agent: (agent as HTMLSelectElement).value }
state.selected = "pending"
close()
render()
}
name.addEventListener("keydown", (e) => (e as KeyboardEvent).key === "Enter" && go())
const el = h(
"div",
{ class: "scrim", role: "dialog", "aria-modal": "true" },
h(
"div",
{ class: "sheet sm" },
h("div", { class: "sheet-head" }, h("h2", {}, "Hire a Bot"), h("p", {}, "Give it a name. It gets one long-lived thread in the current Space.")),
h("div", { class: "sheet-body" }, field("Name", name), field("Agent", agent)),
h("div", { class: "sheet-foot" }, h("span", { class: "spacer" }), h("button", { class: "btn", onclick: () => close() }, "Cancel"), h("button", { class: "btn btn-primary", onclick: go }, "Hire")),
),
)
const close = sheet(el)
name.focus()
}
// ------------------------------------------------------- teleport an app
// The SDK's drop zone (`<cua-drop-zone>`) is a plain web component.
defineTeleportElements()
/** "Teleport an app…" ships with Cua Spaces: say so. */
function openTeleportApp(): void {
state.notice = APP_TELEPORT_MESSAGE
renderMain()
}
/** An app dragged from Finder or the Dock onto the Space: say app teleport
* ships with Cua Spaces instead of uploading the bundle. */
function dropApp(dt: DataTransfer): boolean {
if (classifyBrowserDrop(readDataTransfer(dt), []).kind !== "app") return false
openTeleportApp()
return true
}
// --------------------------------------------------------------- computer
const canvas = h("canvas", { width: 1280, height: 800 }) as HTMLCanvasElement
let current: { ws: { close(code?: number): void; send?(d: string): void }; close: () => Promise<void> } | null = null
let streamedSpace: string | null = null
let windowList: WindowRow[] = []
let zoneStatus: DropZoneStatus = idleDropZone
const desktopPainter = painter(canvas, (frames, w, hgt, first) => {
state.stream.frames = frames
state.stream.width = w
state.stream.height = hgt
if (first || performance.now() - statsTick > 1000) {
statsTick = performance.now()
renderComputer()
}
})
// Picture in picture (`@trycua/cua/spaces/pip`): Document PiP when the
// browser has it, video PiP otherwise; one at a time, of the desktop or of
// one window. A window PiP has its own stream, closed with the PiP.
const pip = new PictureInPicture()
let windowStream: { key: string; close: () => void } | null = null
pip.subscribe((src) => {
if (windowStream && (!src || pipKey(src) !== windowStream.key)) {
windowStream.close()
windowStream = null
}
renderComputer()
})
function toggleDesktopPip(): void {
pip.toggle(desktopSource, canvas, { title: state.snap.space?.name ?? "Desktop" }).catch(fail)
}
function toggleWindowPip(w: WindowRow): void {
const src = windowSource(w)
if (pip.isOpen(src)) return pip.close()
const size = w.width && w.height ? pipSize(w.width, w.height, 1280) : { width: 1280, height: 800 }
const c = h("canvas", { width: size.width, height: size.height }) as HTMLCanvasElement
// A first (black) frame, so the video fallback has something to show.
c.getContext("2d")?.fillRect(0, 0, c.width, c.height)
// Open first (the PiP needs the click), then attach the window's stream.
pip
.open(src, c, { title: pipLabel(src) })
.then(async () => {
const t = await api<OpenedMedia & { needsHeaders: boolean }>("stream", { body: { maxFps: 30, maxDimension: 1280, windowId: w.windowId } })
const paint = painter(c, () => {})
const { ws } = await attachMedia(WebSocket as never, t, 20_000, paint.feed)
paint.socket = ws as never
const close = () => {
ws.close(1000)
paint.reset()
void api("stream/close", { body: { mediaSessionId: t.mediaSessionId } }).catch(() => {})
}
if (pip.isOpen(src)) windowStream = { key: pipKey(src), close }
else close()
})
.catch((e) => {
if (pip.isOpen(src)) pip.close()
fail(e)
})
}
async function refreshWindows(): Promise<void> {
const space = state.snap.space
if (!space || !space.features.includes("window_stream")) {
windowList = []
return
}
const r = await api<{ windows: WindowRow[] }>("windows").catch(() => ({ windows: [] as WindowRow[] }))
const before = JSON.stringify(windowList)
windowList = r.windows
if (JSON.stringify(windowList) !== before) renderComputer()
}
setInterval(() => {
if (state.computerOpen && state.snap.space) void refreshWindows()
}, 5000)
// The one drop zone: files upload (SHA-256 verified); an app bundle or
// "Teleport an app…" shows that app teleport ships with Cua Spaces.
const fileInput = h("input", {
type: "file",
multiple: true,
hidden: true,
onchange: (e: Event) => {
const input = e.target as HTMLInputElement
const files = Array.from(input.files ?? [])
input.value = ""
void sendFiles(files)
},
}) as HTMLInputElement
document.body.append(fileInput)
async function sendFiles(files: File[]): Promise<void> {
if (!files.length) return
const run = state.snap.bots.some((b) => b.runId === state.selected) ? state.selected : null
zoneStatus = { kind: "sending", label: sendingLabel(files.map((f) => f.name)) }
renderComputer()
const sent: DropZoneFile[] = []
for (const f of files) {
const r = await attachFile(f, run, true)
if (!r) {
zoneStatus = { kind: "failed", message: `${f.name} was not sent` }
return renderComputer()
}
sent.push({ name: f.name, bytes: r.bytes, dest: r.dest })
}
zoneStatus = { kind: "sent", files: sent }
renderComputer()
}
function renderComputer(): void {
const space = state.snap.space
const streams = !!space && space.features.includes("desktop_stream")
const live = current && state.stream.frames > 0
const overlayText = !space ? "No Space yet" : !streams ? "This Space does not stream its desktop" : !state.stream.on ? "Stream stopped" : state.stream.status || "Connecting to the desktop…"
const screen = h(
"div",
{ class: "screen" },
canvas,
cursorHost,
live
? h("span", { class: "live" }, h("span", { class: "dot" }), `Live · ${state.stream.width}×${state.stream.height}`)
: h("div", { class: "overlay" }, h("img", { src: koalaSleep, alt: "" }), h("div", {}, overlayText)),
)
const body: Array<HTMLElement | null> = [
screen,
h("div", { class: "screen-bar" }, h("span", {}, space ? space.name : "No Space"), live ? h("span", {}, `· ${state.stream.frames} frames`) : null, h("span", { class: "spacer" }), presenceEl),
]
if (space) {
const directUrl = h("input", { class: "input", placeholder: "http://127.0.0.1:3211", id: "direct-url" })
const directToken = h("input", { class: "input", type: "password", placeholder: "spacesd token", id: "direct-token" })
body.push(
renderWindowList({ windows: windowList, isOpen: (src) => pip.isOpen(src), canPip: pip.mode !== "none", onPip: toggleWindowPip }),
renderDropZone({
status: zoneStatus,
onSendFile: () => fileInput.click(),
onTeleportApp: () => openTeleportApp(),
onDrop: (dt) => {
const files = Array.from(dt.files ?? [])
if (!dropApp(dt)) void sendFiles(files)
},
}),
h(
"div",
{ class: "panel-card" },
h("h3", {}, "Space"),
h(
"dl",
{ class: "kv" },
h("dt", {}, "Name"),
h("dd", {}, space.name),
h("dt", {}, "Id"),
h("dd", { class: "selectable", title: space.id }, space.id),
h("dt", {}, "Provider"),
h("dd", {}, space.provider),
),
h(
"details",
{},
h("summary", { class: "hint" }, "Connect the stream directly (wasm gRPC-Web)"),
h(
"div",
{ style: "display:grid;gap:6px;margin-top:8px" },
directUrl,
directToken,
h("button", { class: "btn", onclick: () => void startDirect((directUrl as HTMLInputElement).value.trim(), (directToken as HTMLInputElement).value) }, "Connect"),
),
),
),
)
}
const desktopPip = pip.isOpen(desktopSource)
computer.replaceChildren(
h(
"header",
{ class: "pane-header" },
icon("monitor"),
h("span", { class: "title" }, "Computer"),
h("span", { class: "spacer" }),
h(
"button",
{
class: "icon-btn",
title: state.stream.on ? "Stop the stream" : "Start the stream",
disabled: !streams,
onclick: () => {
state.stream.on = !state.stream.on
if (!state.stream.on) void stopStream()
else streamedSpace = null
syncStream()
},
},
icon(state.stream.on ? "stop" : "play"),
),
h(
"button",
{
class: `icon-btn ${desktopPip ? "on" : ""}`,
title: desktopPip ? "Close picture in picture" : "Picture in picture",
disabled: !live || pip.mode === "none",
onclick: toggleDesktopPip,
},
icon("pip"),
),
h(
"button",
{
class: "icon-btn",
title: "Hide",
onclick: () => {
state.computerOpen = false
app.classList.add("computer-closed")
renderMain()
},
},
icon("close"),
),
),
h("div", { class: "computer-body" }, ...body.filter((n): n is HTMLElement => n !== null)),
)
}
/** H.264 over the media socket into `target` with WebCodecs, resyncing on
* the next keyframe after a decoder error. */
function painter(target: HTMLCanvasElement, onFrame: (frames: number, width: number, height: number, first: boolean) => void) {
const ctx = target.getContext("2d")
let decoder: VideoDecoder | null = null
let frames = 0
const make = (codec: string) => {
const d = new VideoDecoder({
output: (frame) => {
if (target.width !== frame.displayWidth || target.height !== frame.displayHeight) {
target.width = frame.displayWidth
target.height = frame.displayHeight
}
ctx?.drawImage(frame, 0, 0)
frame.close()
},
error: () => {
decoder = null
self.socket?.send?.(encodeControl("request_keyframe"))
},
})
d.configure({ codec, optimizeForLatency: true })
return d
}
const self = {
/** The media socket, once attached (for keyframe requests). */
socket: null as { send?(d: string): void } | null,
feed(s: MediaSocketState, data: ArrayBuffer): void {
const p = parseBinary(data)
if (p.kind !== "video" || p.descriptor.codec !== "h264") return
if (!decoder) {
if (!p.descriptor.keyframe) return
decoder = make(avcCodecString(p.payload) ?? "avc1.42e01f")
}
decoder.decode(new EncodedVideoChunk({ type: p.descriptor.keyframe ? "key" : "delta", timestamp: p.descriptor.capture_timestamp_us ?? 0, data: p.payload }))
const first = frames === 0
frames = s.frames
onFrame(frames, p.descriptor.width_px, p.descriptor.height_px, first)
},
reset(): void {
decoder?.close()
decoder = null
frames = 0
self.socket = null
},
}
return self
}
let statsTick = 0
async function startStream(open: () => Promise<{ media: OpenedMedia; close: () => Promise<void> }>): Promise<void> {
await stopStream()
if (typeof VideoDecoder === "undefined") throw new Error("this browser has no WebCodecs VideoDecoder")
state.stream.status = "Connecting to the desktop…"
const { media, close } = await open()
const { ws } = await attachMedia(WebSocket as never, media, 20_000, desktopPainter.feed)
desktopPainter.socket = ws as never
current = { ws: ws as never, close }
}
async function stopStream(): Promise<void> {
const c = current
current = null
desktopPainter.reset()
state.stream.frames = 0
if (pip.isOpen(desktopSource)) pip.close()
if (c) {
c.ws.close(1000)
await c.close().catch(() => {})
}
}
/** Streams the selected Space through a server ticket (the spacesd token
* never reaches the page). */
function syncStream(): void {
const space = state.snap.space
const want = space && state.stream.on && space.features.includes("desktop_stream") ? space.id : null
if (want === streamedSpace) return renderComputer()
streamedSpace = want
windowList = []
pip.close()
void refreshWindows()
if (!want) {
void stopStream().then(renderComputer)
return
}
startStream(async () => {
const t = await api<OpenedMedia & { needsHeaders: boolean }>("stream", { body: { maxFps: 30, maxDimension: 1280 } })
if (t.needsHeaders) throw new Error("this Space's media socket needs gateway headers a browser cannot send")
return { media: t, close: () => api("stream/close", { body: { mediaSessionId: t.mediaSessionId } }).then(() => {}) }
})
.catch((e) => {
state.stream.status = `Stream unavailable: ${e instanceof Error ? e.message : String(e)}`
})
.finally(renderComputer)
}
/** Direct: the wasm gRPC-Web client opens the session itself. */
async function startDirect(url: string, envToken: string): Promise<void> {
try {
streamedSpace = "direct"
await startStream(async () => {
const sdk = await import("@trycua/cua/browser")
await sdk.initialize()
const env = (await sdk.Cua.embedded().spacesd(url, envToken || undefined)) as unknown as JsonRpc
const media = await openDesktopMedia(env, url, { maxFps: 30, maxDimension: 1280 })
return { media, close: () => closeMedia(env, media.mediaSessionId) }
})
} catch (e) {
fail(e)
}
renderComputer()
}
// Presence: join as the operator when the Space has the service; the
// cursor follows the pointer over the stream (throttled), and the others'
// cursors are drawn on it by the SDK's PresenceView, fed the server's raw
// presence events: interpolated, shaped, faded when idle, and removed when a
// heartbeat stops listing them or an agent's run ends.
const presenceEl = h("span", { class: "presence" })
const cursorHost = h("div", { class: "cursor-host" })
let presenceView: PresenceView | null = null
function drawCursors(p: { me: string | null; members?: PresenceMember[] } | null | undefined): void {
const me = p?.me ?? null
if (!me) {
presenceView = null
} else if (presenceView?.me !== me) {
const mine = p?.members?.find((m) => m.participant.participantId === me)
presenceView = mine ? PresenceView.from(mine.participant, p?.members ?? []) : null
}
if (!presenceView) cursorHost.replaceChildren()
}
// A timer rather than rAF alone, so staleness still expires in a hidden tab.
function paintCursors(): void {
if (presenceView) cursorHost.replaceChildren(renderDrawables(presenceView.drawables(Date.now())))
}
setInterval(paintCursors, 1_000)
const frame = () => {
paintCursors()
requestAnimationFrame(frame)
}
requestAnimationFrame(frame)
let presenceSpace: string | null = null
async function syncPresence(): Promise<void> {
const space = state.snap.space
const want = space?.features.includes("presence") ? space.id : null
if (want === presenceSpace) return
presenceSpace = want
presenceEl.replaceChildren()
drawCursors(null)
if (want) await api("presence/join", { body: { displayName: "Operator" } }).then(refreshPresence, () => {})
}
async function refreshPresence(): Promise<void> {
const r = await api<{ me: string | null; entries: CursorView[]; cursors: CursorView[]; colors?: Record<string, string>; members?: PresenceMember[] }>("presence")
drawCursors(r)
applyAssignedColors(r.colors)
presenceEl.replaceChildren(
...r.entries.map((m) =>
h(
"span",
{ class: "avatar initials", style: `background:${m.color || "#5d5d63"}`, title: `${m.displayName}${m.agent ? " (agent)" : ""}${m.participantId === r.me ? " (you)" : ""}` },
m.displayName
.split(/\s+/)
.map((w) => w[0] ?? "")
.join("")
.slice(0, 2)
.toUpperCase(),
),
),
)
}
let lastCursor = 0
canvas.addEventListener("pointermove", (e) => {
const now = performance.now()
if (!presenceSpace || now - lastCursor < 100) return
lastCursor = now
const r = canvas.getBoundingClientRect()
void api("presence/cursor", { body: { x: (e.clientX - r.left) / r.width, y: (e.clientY - r.top) / r.height } }).catch(() => {})
})
// ------------------------------------------------------------------ events
/** Avatars follow the colors the server assigned (the Bots' cursors). */
function applyAssignedColors(colors: Record<string, string> | undefined): void {
if (colors && setAssignedColors(colors)) {
renderSidebar()
renderMain()
}
}
function render(): void {
renderSidebar()
renderMain()
renderComputer()
}
function connectEvents(): void {
const proto = location.protocol === "https:" ? "wss" : "ws"
const ws = new WebSocket(`${proto}://${location.host}/events?token=${encodeURIComponent(token)}`)
ws.onopen = () => {
state.connected = true
renderSidebar()
}
ws.onmessage = (ev) => {
const m = JSON.parse(String(ev.data)) as {
type: string
state?: Snapshot
event?: unknown
cursors?: CursorView[]
colors?: Record<string, string>
}
if (m.type === "presence") {
// Cursor moves redraw the pointers; joins and leaves refresh the avatars too.
if (m.event && presenceView) presenceView.apply(m.event as unknown as PresenceEvent, Date.now())
applyAssignedColors(m.colors)
if ((m.event as { kind?: string } | undefined)?.kind !== "cursor_moved") void refreshPresence().catch(() => {})
}
if (m.type !== "state" || !m.state) return
const spaceChanged = m.state.space?.id !== state.snap.space?.id
state.snap = m.state
if (spaceChanged) {
state.selectedSpace = m.state.space?.id ?? null
void refreshSpaces().catch(() => {})
syncStream()
void syncPresence()
}
if (m.state.presence?.colors) setAssignedColors(m.state.presence.colors)
if (!state.selected && state.snap.bots[0]) state.selected = state.snap.bots[0].runId
renderSidebar()
renderMain()
}
ws.onclose = () => {
state.connected = false
renderSidebar()
setTimeout(connectEvents, 2000)
}
}
render()
if (!token) fail("Open the URL the server printed (it carries #token=…).")
else {
api<{ cloud: boolean }>("config")
.then((c) => {
state.cloud = c.cloud
})
.catch(() => {})
.finally(() => {
connectEvents()
refreshSpaces()
.then(() => {
if (params.get("sheet") === "wizard") openWizard(params.get("step") ? { step: Number(params.get("step")) } : undefined)
})
.catch(fail)
})
}