Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Updated types
Dillon DuPont52f7eda6e7b9
diff --git a/libs/typescript/agent/src/types.ts b/libs/typescript/agent/src/types.tsindex e5c079f02e..48a4bc3deb 100644--- a/libs/typescript/agent/src/types.ts+++ b/libs/typescript/agent/src/types.ts@@ -1,3 +1,9 @@+// #region Request+export type ConnectionType = 'http' | 'https' | 'peer';+export interface AgentClientOptions {+ timeout?: number;+ retries?: number;+}// Request types matching the Python proxy APIexport interface AgentRequest {model: string;@@ -13,7 +19,28 @@ export interface AgentRequest {[key: string]: any;};}+// #endregion++++// #region Response+// Response types+export interface AgentResponse {+ output: AgentMessage[];+ usage: Usage;+}+// Usage information+export interface Usage {+ prompt_tokens: number;+ completion_tokens: number;+ total_tokens: number;+ response_cost: number;+}+// #endregion+++// #region Messages// Agent message types - can be one of several different message typesexport type AgentMessage =| UserMessage@@ -21,110 +48,132 @@ export type AgentMessage =| ReasoningMessage| ComputerCallMessage| ComputerCallOutputMessage;--// User input message+// Input messageexport interface UserMessage {- role: 'user';- content: string;+ type?: 'message';+ role: 'user' | 'system' | 'developer';+ content: string | InputContent[];}--// Assistant response message+// Output messageexport interface AssistantMessage {type: 'message';role: 'assistant';content: OutputContent[];}--// Reasoning/thinking message+// Output reasoning/thinking messageexport interface ReasoningMessage {type: 'reasoning';summary: SummaryContent[];}--// Computer action call+// Output computer action callexport interface ComputerCallMessage {type: 'computer_call';call_id: string;status: 'completed' | 'failed' | 'pending';action: ComputerAction;}--// Computer action output (usually screenshot)+// Output computer action result (always a screenshot)export interface ComputerCallOutputMessage {type: 'computer_call_output';call_id: string;- output: InputContent;+ output: ComputerResultContent;}+// #endregion+-// Content types++// #region Message Content+export interface InputContent {+ type: 'input_image' | 'input_text';+ text?: string;+ image_url?: string;+}export interface OutputContent {type: 'output_text';text: string;}-export interface SummaryContent {type: 'summary_text';text: string;}--export interface InputContent {- type: 'input_image' | 'input_text';- text?: string;- image_url?: string;+export interface ComputerResultContent {+ type: 'computer_screenshot' | 'input_image';+ image_url: string;}+// #endregion-// Computer action types+++// #region Actionsexport type ComputerAction =+ | ComputerActionOpenAI+ | ComputerActionAnthropic;+// OpenAI Computer Actions+export type ComputerActionOpenAI =| ClickAction- | TypeAction- | KeyAction+ | DoubleClickAction+ | DragAction+ | KeyPressAction+ | MoveAction+ | ScreenshotAction| ScrollAction+ | TypeAction| WaitAction;-export interface ClickAction {type: 'click';- coordinate: [number, number];+ button: 'left' | 'right' | 'wheel' | 'back' | 'forward';+ x: number;+ y: number;}--export interface TypeAction {- type: 'type';- text: string;+export interface DoubleClickAction {+ type: 'double_click';+ button?: 'left' | 'right' | 'wheel' | 'back' | 'forward';+ x: number;+ y: number;}--export interface KeyAction {- type: 'key';- key: string;+export interface DragAction {+ type: 'drag';+ button?: 'left' | 'right' | 'wheel' | 'back' | 'forward';+ path: Array<[number, number]>;+}+export interface KeyPressAction {+ type: 'keypress';+ keys: string[];+}+export interface MoveAction {+ type: 'move';+ x: number;+ y: number;+}+export interface ScreenshotAction {+ type: 'screenshot';}-export interface ScrollAction {type: 'scroll';- coordinate: [number, number];- direction: 'up' | 'down' | 'left' | 'right';+ scroll_x: number;+ scroll_y: number;+ x: number;+ y: number;+}+export interface TypeAction {+ type: 'type';+ text: string;}-export interface WaitAction {type: 'wait';- seconds?: number;-}--// Usage information-export interface Usage {- prompt_tokens: number;- completion_tokens: number;- total_tokens: number;- response_cost: number;-}--// Response types-export interface AgentResponse {- output: AgentMessage[];- usage: Usage;-}--// Connection types-export type ConnectionType = 'http' | 'https' | 'peer';--export interface AgentClientOptions {- timeout?: number;- retries?: number;}+// Anthropic Computer Actions+export type ComputerActionAnthropic =+ | LeftMouseDownAction+ | LeftMouseUpAction;+export interface LeftMouseDownAction {+ type: 'left_mouse_down';+ x: number;+ y: number;+}+export interface LeftMouseUpAction {+ type: 'left_mouse_up';+ x: number;+ y: number;+}+// #endregion\ No newline at end of file