Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

19.2 KB
Raw Blame
// SPDX-License-Identifier: FSL-1.1-MIT
// Copyright (c) 2026 Cua AI, Inc.

import CuaSpaces
import CuaSpacesStreaming
#if canImport(AppKit)
import AppKit
import Combine
import Foundation
import SwiftUI

/// Everything the running app holds: the backend it attached to, the live
/// `BotStore` behind every screen, the one shared `LiveStreamSession` the Agent
/// Computer tiers and the PiP all observe, and where the user currently is.
///
/// There is deliberately **one** of these. The app model is one persistent Space per
/// account with roughly fifty Bots inside it, so the app has one store, one
/// stream session, and one navigation stack, not one of anything per Bot.
@MainActor
final class AppModel: ObservableObject {

    // MARK: Where the user is

    /// The app's navigation. Flat and value-typed rather than a
    /// `NavigationStack` path, because the two surfaces (mobile canvas and
    /// desktop shell) show the *same* route differently and a shared enum is
    /// the only thing both can read.
    enum Route: Hashable {
        /// The roster: home.
        case roster
        /// One Bot's single long-lived thread.
        case thread(String)
        /// Agent Computer tier 2: the preview pinned above the composer.
        case preview(String)
        /// Agent Computer tier 3: the full-screen takeover.
        case takeover(String)

        /// A Bot's Routines surface: recurring tasks it runs on a schedule.
        /// Carries a **bot** id.
        case routines(String)
        /// A thread with more than one Bot in it. Carries a **chat** id, not a
        /// bot id: a group has members rather than an owner, which is why this
        /// is the one route whose payload `botID` deliberately does not answer.
        case groupThread(String)

        var botID: String? {
            switch self {
            case .roster, .groupThread: return nil
            case .thread(let id), .preview(let id), .takeover(let id),
                 .routines(let id): return id
            }
        }

        /// The group chat this route is about, if it is one.
        var chatID: String? {
            if case .groupThread(let id) = self { return id }
            return nil
        }
    }

    /// Which surface the window is showing. Both are real surfaces over
    /// the same store, which is the point: they are checked separately
    /// because their layouts genuinely differ.
    enum Surface: String, CaseIterable, Identifiable {
        case mobile = "Mobile"
        case desktop = "Desktop"
        var id: String { rawValue }
    }

    @Published var signedIn = false
    @Published var route: Route = .roster
    @Published var surface: Surface = .mobile
    /// `system`, `light` or `dark` (`AppAppearance`). The window follows
    /// macOS unless the user picks one.
    @Published var desktopTheme: String = AppAppearance.system.rawValue
    /// The New Space wizard is open.
    @Published var creatingSpace = false
    /// Spaces this app created, newest first, for the sidebar and the status
    /// line. The roster keeps running on the Space the app attached to.
    @Published private(set) var createdSpaces: [String] = []
    /// The sidebar is showing.
    @Published var sidebarVisible = true
    /// A hire sheet is open over the roster.
    @Published var hiring = false
    /// A `NewGroupSheet` is open over the roster.
    @Published var creatingGroup = false
    /// The last thing that did not go through, shown under the composer.
    @Published var lastRefusal: String?
    /// What the backend attachment is doing, for the status strip.
    @Published private(set) var backendNote: String = "not connected"

    // MARK: What the app is made of

    let store: BotStore
    /// The app's single `RoutineStore`. One per app, not one per Bot: routines
    /// are persisted to a single file and fired by a single scheduler, and a
    /// store per Bot would mean a scheduler per Bot racing over that file.
    let routines = RoutineStore()
    /// The app's single `GroupChatStore`, for the same reason.
    let groups = GroupChatStore()
    let pip = StreamPiPController()
    /// The desktop shell's chrome: which pane is open, whether the header is
    /// in compose mode, which overlay is up. One per app, beside the store.
    let shell = ShellState()
    /// The one rcdp session every Agent Computer surface observes. `nil` until
    /// a Space with a streamable window is attached. The app is usable
    /// without it, it just has no live pixels.
    @Published private(set) var session: LiveStreamSession?
    /// Pop-outs of single Space windows, each on a stream of its own. `nil`
    /// until a Space is attached.
    @Published private(set) var windowPiPs: StreamPiPSet?
    /// App icons for the window list, looked up in the same Space.
    @Published private(set) var windowIcons: WindowIcons?

    private let client: SpacesClient
    /// The concrete MCP client, when there is one. Needed for `local_rcdp`,
    /// which is not in the `SpacesClient` protocol.
    private let mcp: SDKSpacesClient?
    let isLiveBackend: Bool

    // MARK: Construction

    init() {
        let (client, mcp, note) = Self.resolveBackend()
        self.client = client
        self.mcp = mcp
        self.isLiveBackend = mcp != nil
        self.store = BotStore(client: client, rosterFile: BotStore.rosterFileURL())
        self.backendNote = note
    }

    /// Choose a backend without ever creating a sandbox.
    ///
    /// The app attaches to the Space named by `OPENKOALABOTS_TEST_SPACE` through
    /// the cua SDK (a running `cua daemon`, or the Spaces runtime embedded in
    /// this process, `OPENKOALABOTS_SPACES=daemon|embedded`), and otherwise runs
    /// against `DemoSpacesClient` so that launching the app is never
    /// destructive and never fails. It deliberately does **not** fall through
    /// to `create_space` on its own: that can create a metered cloud sandbox
    /// (`FRICTION.md` §5), which is not something double-clicking an app should
    /// ever do.
    private static func resolveBackend() -> (SpacesClient, SDKSpacesClient?, String) {
        guard SDKSpacesClient.overriddenSpace != nil else {
            return (DemoSpacesClient(), nil,
                    "offline: set \(SDKSpacesClient.spaceOverrideVariable) to attach to a Space")
        }
        do {
            // Pure construction: no handshake and no subprocess. The first
            // real call (made from `start()`, which is `async`) pays for the
            // connection off the main actor (`FRICTION.md` §30, §47).
            let c = try SDKSpacesClient(backend: SDKSpacesClient.backendFromEnvironment())
            return (c, c, "live")
        } catch {
            return (DemoSpacesClient(), nil, "offline, the cua SDK did not start: \(error)")
        }
    }

    // MARK: Lifecycle

    private var started = false

    /// Attach, read the roster back, and start streaming if there is anything
    /// to stream. Safe to call more than once.
    func start() async {
        guard !started else { return }
        started = true
        await store.connect()

        // Routines and group chats both send *through* the roster store, so
        // they can only be attached once `BotStore.connect()` has decided what
        // is behind it. Before this point a fired routine would have nowhere to
        // go; the scheduler is therefore started here and not in `init`.
        routines.load()
        routines.attach(runner: BotStoreRoutineRunner(store: store))
        routines.startScheduler()
        groups.attach(messenger: BotStoreGroupMessenger(store: store))

        if case .attached(let space) = store.connection {
            backendNote = isLiveBackend ? "attached to \(space)" : backendNote
            await store.refreshWindows()
            attachStream(space: space)
        }
        store.startPolling(every: .seconds(3))
    }

    /// Shut down everything this app started. It never deletes the Space:
    /// the app attaches to a Space it does not own.
    func shutDown() async {
        routines.stopScheduler()
        _ = routines.save()
        _ = store.saveRosterIdentities()
        store.stopPolling()
        pip.popIn()
        windowPiPs?.popInAll()
        await session?.stop()
    }

    // MARK: Sign in

    /// The sign-in gate. Entirely local (there is no account system behind
    /// this sample), but it is a real gate: nothing but `SignInScreen` is
    /// reachable until it passes.
    func signIn() {
        signedIn = true
        route = .roster
    }

    // MARK: Streaming

    /// Build the one shared session over the Space's own frames.
    ///
    /// This used to pull `local_rcdp` apart by hand, rebuild an endpoint, and
    /// hold the two closures in an adapter: the three layers `FRICTION.md`
    /// §10 records as existing only to route around a missing capability. The
    /// SDK has the capability now: `Space.streamEndpoint()` is "hand me its
    /// frames", kept distinct from `Space.present(_:)`, which is "show this to
    /// the operator" and would draw on the wrong machine.
    private func attachStream(space: String) {
        guard let mcp else { return }
        Task { [weak self] in
            guard let handle = try? await mcp.sdkSpace(space) else { return }
            await MainActor.run {
                guard let self else { return }
                let s = LiveStreamSession(space: handle)
                self.session = s
                self.windowPiPs = StreamPiPSet(provider: SpaceStreamProvider(space: handle))
                self.windowIcons = WindowIcons(space: handle)
                // The shared presence cursor: joins once the stream is open,
                // and a Space without presence still streams.
                Task { await s.joinPresence(as: Self.presenceName) }
                // Avatars follow the colors the server assigned.
                self.presenceColors = s.$participants
                    .combineLatest(s.$localParticipant)
                    .receive(on: DispatchQueue.main)
                    .sink { others, me in
                        PresenceColorBook.shared.update(others + (me.map { [$0] } ?? []))
                    }
                Task { await s.refreshWindows() }
            }
        }
    }

    private var presenceColors: AnyCancellable?

    /// This operator's name on the Space's presence roster.
    static let presenceName = ProcessInfo.processInfo.environment["OPENKOALABOTS_PRESENCE_NAME"] ?? "Operator"

    /// What an Agent Computer tier should draw. `.fixture` when there is
    /// no Space behind the app, which keeps every surface usable offline.
    var screenSource: AgentScreenSource {
        session.map { .live($0) } ?? .fixture
    }

    /// Pop the stream out into a floating panel that survives navigation. This
    /// is what makes the PiP "available throughout": it is an `NSPanel` owned
    /// by the model, not a view inside any route.
    func togglePiP() {
        guard let session else { return }
        pip.toggle(session: session)
    }

    /// Pop one Space window out into a floating panel of its own. It streams
    /// that window only, beside the desktop pop-out, until its panel closes.
    func toggleWindowPiP(_ window: StreamWindow) {
        windowPiPs?.toggle(.window(window))
    }

    // MARK: Navigation

    func open(_ route: Route) {
        self.route = route
        lastRefusal = nil
        // Tell the poll loop which transcript is actually on screen, so it
        // spends its one per-tick `agent_status` on that Bot rather than on all
        // of them (`BotStore.startPolling`, `FRICTION.md` §51).
        store.focus(route.botID)
        if let id = route.botID { Task { await store.refresh(id) } }
    }

    /// Back out one level: takeover → thread → roster.
    func back() {
        switch route {
        case .roster: break
        case .groupThread:
            route = .roster
        case .thread:
            route = .roster
        case .routines(let id):
            // Routines is pushed *from* a thread, so backing out returns to it
            // rather than skipping a level to the roster.
            route = .thread(id)
        case .preview(let id), .takeover(let id):
            route = .thread(id)
        }
        lastRefusal = nil
    }

    /// Walk *up* the Agent Computer's three tiers for one Bot.
    ///
    /// The tiers are an escalation, not three destinations: tier 1 is the
    /// status glyph already inline in the transcript, tier 2 pins a preview
    /// above the composer, tier 3 takes the whole screen. The monitor button in
    /// the thread header is the escalator, and it wraps back to the thread so
    /// one control can walk the whole ladder in both directions.
    func escalate(from botID: String) {
        switch route {
        case .thread(botID):   open(.preview(botID))
        case .preview(botID):  open(.takeover(botID))
        case .takeover(botID): open(.thread(botID))
        default:               open(.preview(botID))
        }
    }

    /// The Bot the current route is about, falling back to the first in the
    /// roster so the desktop shell always has something to show.
    var focusedBot: Bot? {
        if let id = route.botID, let b = store.bot(id) { return b }
        return store.bots.first
    }

    // MARK: Talking to a Bot

    /// Send, and show the answer either way.
    ///
    /// A mid-turn refusal is correct behaviour, not an error, and it is exactly
    /// the thing this app must not swallow (`FRICTION.md` §24): it lands in
    /// `lastRefusal` and is drawn under the composer, as well as in the
    /// transcript and `store.notices`, which the store already does.
    func send(_ text: String, to botID: String) {
        Task {
            let outcome = await store.send(text, to: botID)
            lastRefusal = outcome.accepted
                ? nil
                : "Not delivered: \(outcome.reason.isEmpty ? "refused" : outcome.reason)"
        }
    }

    func stop(_ botID: String) { Task { await store.stop(botID) } }

    // MARK: Creating a Space

    /// What `Create` in the New Space wizard calls. The live client when the
    /// app is attached through the SDK; otherwise a client built on demand,
    /// because creating a Space is an explicit request and does not need the
    /// app to be attached to one first. Tests inject a fake.
    var spaceCreator: SpaceCreating?

    private func resolvedCreator() throws -> SpaceCreating {
        if let spaceCreator { return spaceCreator }
        if let mcp { spaceCreator = mcp; return mcp }
        let c = try SDKSpacesClient(backend: SDKSpacesClient.backendFromEnvironment())
        spaceCreator = c
        return c
    }

    /// Create the Space a wizard plan describes, through the SDK. With
    /// `openWhenReady`, its desktop opens in the floating viewer.
    @discardableResult
    func createSpace(_ plan: SpacePlan) async throws -> String {
        guard plan.isValid else {
            throw SpaceCreationError.invalidPlan(plan.systemError ?? plan.nameError ?? "invalid plan")
        }
        let creator = try resolvedCreator()
        let id = try await SpaceCreationFlow.create(plan, with: creator)
        createdSpaces.insert(id, at: 0)
        creatingSpace = false
        if plan.openWhenReady, let sdk = creator as? SDKSpacesClient {
            openDesktop(of: id, via: sdk)
        }
        return id
    }

    private func openDesktop(of space: String, via sdk: SDKSpacesClient) {
        Task { [weak self] in
            guard let handle = try? await sdk.sdkSpace(space) else { return }
            await MainActor.run {
                guard let self else { return }
                let s = LiveStreamSession(space: handle)
                Task { await s.refreshWindows() }
                self.pip.toggle(session: s)
            }
        }
    }

    // MARK: Group chats

    /// Open a freshly created group. Called back by `NewGroupSheet`.
    func opened(_ chat: GroupChat) {
        creatingGroup = false
        open(.groupThread(chat.id))
    }

    // MARK: Hiring

    /// The shapes and colours a new Bot is drawn from, so a hired Bot looks
    /// like it belongs on the roster rather than like a placeholder.
    static let paletteShapes: [BlobShape] =
        [.circle, .teardrop, .cloud, .hexagon, .egg, .lozenge, .squircle, .triangle, .arch]
    static let paletteColors: [UInt32] =
        [0x8B5CF6, 0x2F80F0, 0x18BE4B, 0x11B5A0, 0xF97316, 0xF4234B, 0x5B5BE8, 0xEE3E8F, 0xE0AE09]

    /// Hire a Bot from the UI: mint a roster identity for it, then start its
    /// one long-lived agent thread in the **shared** Space.
    ///
    /// Note what this does *not* do. It does not create a Space. The app
    /// supports roughly fifty Bots per account and they all live in one
    /// persistent VM, so a Space per Bot would be fifty sandboxes per user and
    /// wrong besides: screens are a presentation split, not a security
    /// boundary. `BotStore.hire` starts a run inside the Space the app is
    /// already attached to.
    /// `Create new Bot`, from the sidebar `+` or from the Marketplace's `Add`.
    ///
    /// **This is the path that replaces hiring, and the ordering is the
    /// point.** The agent is started first; only when that resolves does a
    /// roster row exist; then the conversation opens. There is consequently no
    /// moment at which the user can see a Bot that has nothing behind it, and
    /// therefore no state that needs a word like "not hired yet". If the start
    /// fails, no row appears and the failure is shown.
    @discardableResult
    func createBot(named name: String) async -> String? {
        do {
            let bot = try await store.createConversation(
                named: name, dark: desktopTheme != "light")
            hiring = false
            open(.thread(bot.id))
            return bot.id
        } catch {
            lastRefusal = "Could not create \(name): \(error)"
            return nil
        }
    }

    @discardableResult
    func hire(name: String, prompt: String) async -> String? {
        let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines)
        let task = prompt.trimmingCharacters(in: .whitespacesAndNewlines)
        guard !trimmed.isEmpty, !task.isEmpty else { return nil }

        let id = Self.identifier(for: trimmed, taken: Set(store.bots.map(\.id)))
        let index = store.bots.count
        var bot = Persona.bot(id: id, name: trimmed,
                              dark: desktopTheme != "light",
                              preview: task, timestamp: BotStore.timestamp(Date()),
                              screenIndex: index)
        bot.preview = task
        store.register(bot)
        do {
            _ = try await store.hire(id, prompt: task)
            hiring = false
            open(.thread(id))
            return id
        } catch {
            lastRefusal = "Could not create \(trimmed): \(error)"
            hiring = false
            open(.thread(id))
            return nil
        }
    }

    /// A stable, readable id from a display name, unique within the roster.
    /// One implementation, in the store, because the id is also what the
    /// persona colour and shape are derived from and two implementations would
    /// eventually disagree about a Bot's colour.
    static func identifier(for name: String, taken: Set<String>) -> String {
        BotStore.identifier(for: name, taken: taken)
    }
}
#endif