Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Updated types

Dillon DuPont52f7eda6e7b9
diff --git a/libs/typescript/agent/src/types.ts b/libs/typescript/agent/src/types.ts
index e5c079f02e..48a4bc3deb 100644
--- a/libs/typescript/agent/src/types.ts
+++ b/libs/typescript/agent/src/types.ts
@@ -1,3 +1,9 @@
+// #region Request
+export type ConnectionType = 'http' | 'https' | 'peer';
+export interface AgentClientOptions {
+ timeout?: number;
+ retries?: number;
+}
// Request types matching the Python proxy API
export interface AgentRequest {
model: string;
@@ -13,7 +19,28 @@ export interface AgentRequest {
[key: string]: any;
};
}
+// #endregion
+
+
+
+// #region Response
+// Response types
+export interface AgentResponse {
+ output: AgentMessage[];
+ usage: Usage;
+}
+// Usage information
+export interface Usage {
+ prompt_tokens: number;
+ completion_tokens: number;
+ total_tokens: number;
+ response_cost: number;
+}
+// #endregion
+
+
+// #region Messages
// Agent message types - can be one of several different message types
export type AgentMessage =
| UserMessage
@@ -21,110 +48,132 @@ export type AgentMessage =
| ReasoningMessage
| ComputerCallMessage
| ComputerCallOutputMessage;
-
-// User input message
+// Input message
export interface UserMessage {
- role: 'user';
- content: string;
+ type?: 'message';
+ role: 'user' | 'system' | 'developer';
+ content: string | InputContent[];
}
-
-// Assistant response message
+// Output message
export interface AssistantMessage {
type: 'message';
role: 'assistant';
content: OutputContent[];
}
-
-// Reasoning/thinking message
+// Output reasoning/thinking message
export interface ReasoningMessage {
type: 'reasoning';
summary: SummaryContent[];
}
-
-// Computer action call
+// Output computer action call
export interface ComputerCallMessage {
type: 'computer_call';
call_id: string;
status: 'completed' | 'failed' | 'pending';
action: ComputerAction;
}
-
-// Computer action output (usually screenshot)
+// Output computer action result (always a screenshot)
export interface ComputerCallOutputMessage {
type: 'computer_call_output';
call_id: string;
- output: InputContent;
+ output: ComputerResultContent;
}
+// #endregion
+
-// Content types
+
+// #region Message Content
+export interface InputContent {
+ type: 'input_image' | 'input_text';
+ text?: string;
+ image_url?: string;
+}
export interface OutputContent {
type: 'output_text';
text: string;
}
-
export interface SummaryContent {
type: 'summary_text';
text: string;
}
-
-export interface InputContent {
- type: 'input_image' | 'input_text';
- text?: string;
- image_url?: string;
+export interface ComputerResultContent {
+ type: 'computer_screenshot' | 'input_image';
+ image_url: string;
}
+// #endregion
-// Computer action types
+
+
+// #region Actions
export type ComputerAction =
+ | ComputerActionOpenAI
+ | ComputerActionAnthropic;
+// OpenAI Computer Actions
+export type ComputerActionOpenAI =
| ClickAction
- | TypeAction
- | KeyAction
+ | DoubleClickAction
+ | DragAction
+ | KeyPressAction
+ | MoveAction
+ | ScreenshotAction
| ScrollAction
+ | TypeAction
| WaitAction;
-
export interface ClickAction {
type: 'click';
- coordinate: [number, number];
+ button: 'left' | 'right' | 'wheel' | 'back' | 'forward';
+ x: number;
+ y: number;
}
-
-export interface TypeAction {
- type: 'type';
- text: string;
+export interface DoubleClickAction {
+ type: 'double_click';
+ button?: 'left' | 'right' | 'wheel' | 'back' | 'forward';
+ x: number;
+ y: number;
}
-
-export interface KeyAction {
- type: 'key';
- key: string;
+export interface DragAction {
+ type: 'drag';
+ button?: 'left' | 'right' | 'wheel' | 'back' | 'forward';
+ path: Array<[number, number]>;
+}
+export interface KeyPressAction {
+ type: 'keypress';
+ keys: string[];
+}
+export interface MoveAction {
+ type: 'move';
+ x: number;
+ y: number;
+}
+export interface ScreenshotAction {
+ type: 'screenshot';
}
-
export interface ScrollAction {
type: 'scroll';
- coordinate: [number, number];
- direction: 'up' | 'down' | 'left' | 'right';
+ scroll_x: number;
+ scroll_y: number;
+ x: number;
+ y: number;
+}
+export interface TypeAction {
+ type: 'type';
+ text: string;
}
-
export interface WaitAction {
type: 'wait';
- seconds?: number;
-}
-
-// Usage information
-export interface Usage {
- prompt_tokens: number;
- completion_tokens: number;
- total_tokens: number;
- response_cost: number;
-}
-
-// Response types
-export interface AgentResponse {
- output: AgentMessage[];
- usage: Usage;
-}
-
-// Connection types
-export type ConnectionType = 'http' | 'https' | 'peer';
-
-export interface AgentClientOptions {
- timeout?: number;
- retries?: number;
}
+// Anthropic Computer Actions
+export type ComputerActionAnthropic =
+ | LeftMouseDownAction
+ | LeftMouseUpAction;
+export interface LeftMouseDownAction {
+ type: 'left_mouse_down';
+ x: number;
+ y: number;
+}
+export interface LeftMouseUpAction {
+ type: 'left_mouse_up';
+ x: number;
+ y: number;
+}
+// #endregion
\ No newline at end of file