Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

14.7 KB
Raw Blame
/**
 * OpenKoalaBots's app model on `@trycua/cua/spaces`, the TypeScript twin of the
 * Swift sample's `BotStore` / `SDKSpacesClient`:
 *
 * - attach to **one** Space and run **one long-lived agent thread per Bot**
 *   in it (never a Space per Bot);
 * - a live roster from one `agentList` per tick, joined back to local Bot
 *   identity (runs this app did not start still appear, as unmarked);
 * - each Bot's transcript folded from the run's structured events
 *   (`agentEvents`) by the SDK's `AgentTranscript`, the same rule the Swift
 *   and Tauri samples use: the agent's messages, and its install, tool,
 *   turn-end and other activity as muted groups, each attributed to the
 *   turn that caused it;
 * - a message to a Bot that is mid-turn is *refused* and the refusal is shown,
 *   never silently queued; a failed status probe degrades to `unknown`,
 *   never to health;
 * - attachments go in with `sendFile` (SHA-256 verified), a teleport needs an
 *   explicit approval callback, presence joins as a named participant.
 *
 * Besides types, only the SDK's `AgentTranscript` fold comes from the native
 * library here, so the model runs against a fake Space in unit tests.
 */
import { AgentTranscript } from "@trycua/cua"
import type {
  AgentActionReport,
  AgentRunStatus,
  AgentStartReport,
  PresenceIdentity,
  SpaceInfo,
  SpacePresenceLike,
  SpaceSendFileOptions,
  SpaceSendFileReport,
  TeleportDecision,
  TeleportManifest,
  TeleportReceipt,
} from "@trycua/cua/spaces"

/** The subset of `SpaceLike` the app uses. */
export interface SpacePort {
  id(): string
  info(): SpaceInfo
  supports(feature: string): boolean
  agentStart(agent: string, prompt: string, show: boolean | undefined, options: AgentStartOptions | undefined): Promise<AgentStartReport>
  agentStatus(runId: string, tail: number | undefined): Promise<AgentRunStatus>
  /** The `agent_events` result after `cursor`, as JSON. */
  agentEvents(runId: string, cursor: bigint | undefined, max: number | undefined): Promise<string>
  agentMessage(runId: string, text: string, force: boolean | undefined): Promise<AgentActionReport>
  agentStop(runId: string): Promise<AgentActionReport>
  agentList(): Promise<AgentRunStatus[]>
  sendFile(localPath: string, options: SpaceSendFileOptions): Promise<SpaceSendFileReport>
  teleportManifest(app: string, scope: string | undefined): Promise<TeleportManifest>
  teleport(
    app: string,
    scope: string | undefined,
    approver: { approve(manifest: TeleportManifest): TeleportDecision | undefined },
    /** The id a daemon's two-call teleport resumes with (none here). */
    requestId: string | undefined,
  ): Promise<TeleportReceipt>
  joinPresence(identity: PresenceIdentity, timeoutMs: bigint | undefined): Promise<SpacePresenceLike>
}

/** The SDK's agent options (`SpaceAgentOptions`), every field present. */
export interface AgentStartOptions {
  envFromHost: string[]
  /** More environment for the agent (none here). */
  env: Map<string, string>
  repo: string | undefined
  branch: string | undefined
  cwd: string | undefined
  model: string | undefined
  baseUrl: string | undefined
  exitWhenIdle: boolean | undefined
}

/** A custom model endpoint for every Bot this store starts. */
export interface AgentEndpoint {
  baseUrl?: string
  model?: string
  /** Provider key variables forwarded from this process's environment. */
  envFromHost?: string[]
}

export function agentOptions(e: AgentEndpoint | undefined): AgentStartOptions | undefined {
  if (!e || (!e.baseUrl && !e.model && !e.envFromHost?.length)) return undefined
  return { envFromHost: e.envFromHost ?? [], env: new Map(), repo: undefined, branch: undefined, cwd: undefined, model: e.model, baseUrl: e.baseUrl, exitWhenIdle: undefined }
}

/** The subset of `SpacesLike` the app uses. */
export interface SpacesPort {
  list(): Promise<SpaceInfo[]>
  add(url: string, token: string | undefined, name: string | undefined): Promise<SpaceInfo>
  /** Deletes a created Space's sandbox (a Space added by address is only forgotten). */
  delete_(space: string): Promise<string>
  remove(space: string): Promise<void>
  space(space: string): Promise<SpacePort>
}

export type BotState =
  | "starting"
  | "running"
  | "idle"
  | "awaiting_input"
  | "finished"
  | "failed"
  | "crashed"
  | "stopped"
  | "unknown"

/** States after which a turn has ended. */
export const SETTLED: ReadonlySet<string> = new Set(["idle", "awaiting_input", "finished", "failed", "crashed", "stopped"])

export interface Bot {
  id: string
  name: string
  agent: string
}

/**
 * One item of a thread as the page shows it. `user`: what the user typed
 * (kept locally, never the agent's echo of it); `message`: the agent's words;
 * `activity`: a muted, collapsible group of one-line steps (install, tools,
 * thinking, turn ends, notices, refusals) whose `text` is its summary.
 */
export interface ThreadItem {
  turn: number
  kind: "user" | "message" | "activity"
  text: string
  steps: string[]
}

/** The SDK fold (`AgentTranscript`), as the thread uses it. */
export interface TranscriptFold {
  absorbJson(json: string): bigint | undefined
  note(turn: number, text: string): void
  items(): Array<{ kind: string; turn: number; text: string; steps: string[] }>
  preview(): string | undefined
  revision(): bigint
}

export interface Turn {
  index: number
  message: string
  delivered: boolean
  reason: string
}

export interface RosterEntry {
  runId: string
  bot: Bot | null
  state: BotState
  reason: string
  acceptsMessage: boolean
}

export interface Delivery {
  accepted: boolean
  reason: string
}

function toState(status: string): BotState {
  const known: BotState[] = ["running", "idle", "awaiting_input", "finished", "failed", "crashed", "stopped"]
  return (known as string[]).includes(status) ? (status as BotState) : "unknown"
}

/** Largest event page read per request, and pages read per refresh. */
const EVENT_PAGE = 500
const EVENT_PAGES = 20

/** One Bot's long-lived agent thread in the shared Space. */
export class BotThread {
  readonly turns: Turn[] = []
  state: BotState = "starting"
  reason = ""
  acceptsMessage = false
  /** The run's events, folded by the SDK. Runner turn N is `turns[N - 1]`. */
  private readonly fold: TranscriptFold = new AgentTranscript()
  private cursor = 0n

  private constructor(
    readonly bot: Bot,
    readonly runId: string,
    private readonly space: SpacePort,
  ) {}

  static async hire(space: SpacePort, bot: Bot, prompt: string, options?: AgentStartOptions): Promise<BotThread> {
    // #region docs:ts-agent-start
    const report = await space.agentStart(bot.agent, prompt, false, options)
    const t = new BotThread(bot, report.runId, space)
    // #endregion docs:ts-agent-start
    t.turns.push({ index: 0, message: prompt, delivered: true, reason: "started" })
    // Start notes are setup, like the install progress that follows (turn 0).
    for (const note of report.notes) t.fold.note(0, note)
    return t
  }

  get currentTurn(): number {
    return this.turns.length - 1
  }

  /** Changes whenever the thread's items do. */
  get revision(): string {
    return `${this.turns.length}:${this.fold.revision()}`
  }

  /**
   * The thread: the user's own turns, and the fold's message and activity
   * items (its `user` items are the agent's copy of the prompt, skipped so
   * the prompt is never shown twice), each on the local turn that caused it.
   */
  get items(): ThreadItem[] {
    const out: ThreadItem[] = []
    let next = 0
    const user = (t: Turn) => out.push({ turn: t.index, kind: "user", text: t.message, steps: [] })
    for (const i of this.fold.items()) {
      if (i.kind === "user") continue
      // Install progress (turn 0) follows the first prompt.
      while (next < this.turns.length && this.turns[next].index + 1 <= Math.max(i.turn, 1)) user(this.turns[next++])
      // The SDK's grouping and summaries as they are: setup (turn 0) stays
      // its own group.
      out.push({ turn: Math.max(i.turn - 1, 0), kind: i.kind === "message" ? "message" : "activity", text: i.text, steps: [...i.steps] })
    }
    while (next < this.turns.length) user(this.turns[next++])
    return out
  }

  /** The agent's last message on one line (the roster preview), or "". */
  get preview(): string {
    return this.fold.preview() ?? ""
  }

  /** One refresh: the status, then the events written up to it (so a
   * settled status never leaves its turn's last events unread). A failed
   * probe degrades to `unknown`. */
  async refresh(): Promise<BotState> {
    let s: AgentRunStatus
    try {
      s = await this.space.agentStatus(this.runId, 0)
      for (let i = 0; i < EVENT_PAGES; i++) {
        const json = await this.space.agentEvents(this.runId, this.cursor, EVENT_PAGE)
        const page = JSON.parse(json) as { events?: unknown[]; caught_up?: boolean }
        const next = this.fold.absorbJson(json)
        if (next !== undefined) this.cursor = next
        if (page.caught_up !== false || !page.events?.length) break
      }
    } catch (e) {
      this.state = "unknown"
      this.acceptsMessage = false
      this.reason = `status probe failed: ${e instanceof Error ? e.message : String(e)}`
      return this.state
    }
    this.state = toState(s.status)
    this.reason = s.reason
    this.acceptsMessage = s.acceptsMessage
    return this.state
  }

  /** Polls until the current turn settles, bounded by `maxPolls`. */
  async settle(pollMs: number, maxPolls: number, until?: (t: BotThread) => boolean): Promise<BotState> {
    // A settled turn whose output does not (yet) satisfy `until` gets a few
    // more polls for a lagging tail, then returns rather than spinning.
    let settledPolls = 0
    for (let i = 0; i < maxPolls; i++) {
      await this.refresh()
      if (SETTLED.has(this.state)) {
        // Right after a follow-up turn exits, the runtime can briefly report
        // `crashed` ("recorded no exit status") before the exit code lands;
        // a crash is only believed once it persists.
        const provisional = this.state === "crashed" && settledPolls < 5
        if ((!provisional && (!until || until(this))) || ++settledPolls > 5) return this.state
      }
      await sleep(pollMs)
    }
    throw new Error(`turn ${this.currentTurn} of ${this.bot.name} did not settle after ${maxPolls} polls (state ${this.state})`)
  }

  /** Sends a follow-up. Refused (not queued) unless the Bot is idle. */
  async send(message: string): Promise<Delivery> {
    if (!this.acceptsMessage) {
      const reason = `${this.bot.name} is ${this.state}; a message is refused rather than queued`
      this.fold.note(this.turns.length, `Refused: ${reason}`)
      return { accepted: false, reason }
    }
    // #region docs:ts-agent-message
    const r = await this.space.agentMessage(this.runId, message, false)
    if (!r.ok) {
      this.fold.note(this.turns.length, `Refused: ${r.reason}`)
      return { accepted: false, reason: r.reason }
    }
    // #endregion docs:ts-agent-message
    this.turns.push({ index: this.turns.length, message, delivered: true, reason: r.reason })
    this.state = "running"
    this.acceptsMessage = false
    return { accepted: true, reason: r.reason }
  }

  /** The agent's messages in one turn (activity excluded). */
  outputOf(turn: number): string[] {
    return this.fold
      .items()
      .filter((i) => i.kind === "message" && i.turn === turn + 1)
      .map((i) => i.text)
  }

  async stop(): Promise<AgentActionReport> {
    const r = await this.space.agentStop(this.runId)
    if (r.ok) this.state = "stopped"
    return r
  }
}

/** The app: one Space, many Bots. */
export class BotStore {
  readonly threads = new Map<string, BotThread>()
  /** Every Bot this app knows, hired or not (routines and groups name Bots by id). */
  readonly bots = new Map<string, Bot>()
  roster: RosterEntry[] = []
  /** `agentList` round trips, so "one call per tick" is checkable. */
  rosterCalls = 0

  constructor(
    readonly space: SpacePort,
    readonly endpoint?: AgentEndpoint,
  ) {}

  /** Knows a Bot without starting it. */
  register(bot: Bot): Bot {
    this.bots.set(bot.id, bot)
    return bot
  }

  bot(id: string): Bot | undefined {
    return this.bots.get(id)
  }

  async hire(bot: Bot, prompt: string): Promise<BotThread> {
    this.register(bot)
    const t = await BotThread.hire(this.space, bot, prompt, agentOptions(this.endpoint))
    this.threads.set(t.runId, t)
    return t
  }

  thread(runId: string): BotThread | undefined {
    return this.threads.get(runId)
  }

  /** A Bot's current thread: the one it was hired into last. */
  threadFor(botId: string): BotThread | undefined {
    let last: BotThread | undefined
    for (const t of this.threads.values()) if (t.bot.id === botId) last = t
    return last
  }

  /** One roster tick: one `agentList` whatever the roster size. */
  async refreshRoster(): Promise<RosterEntry[]> {
    this.rosterCalls += 1
    let runs: AgentRunStatus[]
    try {
      runs = await this.space.agentList()
    } catch (e) {
      this.roster = this.roster.map((r) => ({ ...r, state: "unknown", acceptsMessage: false, reason: String(e) }))
      return this.roster
    }
    this.roster = runs.map((r) => ({
      runId: r.runId,
      bot: this.threads.get(r.runId)?.bot ?? null,
      state: toState(r.status),
      reason: r.reason,
      acceptsMessage: r.acceptsMessage,
    }))
    for (const r of runs) {
      const t = this.threads.get(r.runId)
      if (t) {
        t.state = toState(r.status)
        t.acceptsMessage = r.acceptsMessage
        t.reason = r.reason
      }
    }
    return this.roster
  }

  /** Attachments in: SHA-256 verified by the driver, per file. */
  async attach(localPath: string, subdir: string): Promise<SpaceSendFileReport> {
    // #region docs:ts-send-file
    const report = await this.space.sendFile(localPath, {
      targetDirectory: subdir,
      respectIgnoreFiles: true,
      conflict: "rename",
    })
    if (!report.verified) throw new Error(`send_file of ${localPath} was not verified`)
    // #endregion docs:ts-send-file
    return report
  }

  /**
   * Teleports an app session. `decide` sees exactly what would leave this
   * machine and returns the human's decision, or `undefined` to cancel.
   * Works through the Cua Spaces daemon; an embedded runtime refuses with
   * `HostCapabilityMissing` (teleport ships with Cua Spaces).
   */
  async teleport(
    app: string,
    decide: (manifest: TeleportManifest) => TeleportDecision | undefined,
    scope?: string,
  ): Promise<{ receipt: TeleportReceipt; shown: TeleportManifest[] }> {
    const shown: TeleportManifest[] = []
    // #region docs:ts-teleport
    const receipt = await this.space.teleport(app, scope, {
      approve: (m) => {
        shown.push(m)
        return decide(m)
      },
    }, undefined)
    // #endregion docs:ts-teleport
    return { receipt, shown }
  }
}

export function sleep(ms: number): Promise<void> {
  return new Promise((r) => setTimeout(r, ms))
}