Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

agent fixes for scaling resolution on gemini + prefxing issues (#1117)

Sarina Lia5495b294c02
diff --git a/docs/content/docs/cua/reference/agent-sdk/index.mdx b/docs/content/docs/cua/reference/agent-sdk/index.mdx
index 27a291ef95..7ab38a8ff6 100644
--- a/docs/content/docs/cua/reference/agent-sdk/index.mdx
+++ b/docs/content/docs/cua/reference/agent-sdk/index.mdx
@@ -7,7 +7,7 @@ description: Python API reference for building computer-use agents
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/python/agent/agent
- Version: 0.7.27
+ Version: 0.7.29
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.7","href":"/cua/reference/agent-sdk","isCurrent":true},{"version":"0.6","href":"/cua/reference/agent-sdk/v0.6/api","isCurrent":false},{"version":"0.5","href":"/cua/reference/agent-sdk/v0.5/api","isCurrent":false},{"version":"0.4","href":"/cua/reference/agent-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/agent-sdk/v0.3/api","isCurrent":false}]}
currentVersion="0.7"
- fullVersion="0.7.27"
+ fullVersion="0.7.29"
packageName="cua-agent"
/>
diff --git a/docs/content/docs/cua/reference/cli/index.mdx b/docs/content/docs/cua/reference/cli/index.mdx
index 352721ac38..e24add0bfe 100644
--- a/docs/content/docs/cua/reference/cli/index.mdx
+++ b/docs/content/docs/cua/reference/cli/index.mdx
@@ -7,7 +7,7 @@ description: Python API reference for the Cua command-line interface
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/python/cua-cli/cua_cli
- Version: 0.1.5
+ Version: 0.1.6
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.1","href":"/cua/reference/cli","isCurrent":true}]}
currentVersion="0.1"
- fullVersion="0.1.5"
+ fullVersion="0.1.6"
packageName="cua-cli"
/>
diff --git a/docs/content/docs/cua/reference/computer-sdk/index.mdx b/docs/content/docs/cua/reference/computer-sdk/index.mdx
index a8b5c57143..89d2d7e013 100644
--- a/docs/content/docs/cua/reference/computer-sdk/index.mdx
+++ b/docs/content/docs/cua/reference/computer-sdk/index.mdx
@@ -7,7 +7,7 @@ description: Python API reference for controlling virtual machines and computer
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/python/computer/computer
- Version: 0.5.14
+ Version: 0.5.16
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.5","href":"/cua/reference/computer-sdk","isCurrent":true},{"version":"0.4","href":"/cua/reference/computer-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/computer-sdk/v0.3/api","isCurrent":false}]}
currentVersion="0.5"
- fullVersion="0.5.14"
+ fullVersion="0.5.16"
packageName="cua-computer"
/>
@@ -559,707 +559,707 @@ the returned :class:`PtyHandle` reflects any resizes since creation.
---
-## diorama_computer
+## tracing
+
+Computer tracing functionality for recording sessions.
+
+This module provides a Computer.tracing API inspired by Playwright's tracing functionality,
+allowing users to record computer interactions for debugging, training, and analysis.
---
-## Key
+## ComputerTracing
-*Inherits from: Enum*
+Computer tracing class that records computer interactions and saves them to disk.
-Keyboard keys that can be used with press_key.
+This class provides a flexible API for recording computer sessions with configurable
+options for what to record (screenshots, API calls, video, etc.).
-These key names follow a consistent cross-platform keyboard key naming convention.
+### Constructor
+
+```python
+ComputerTracing(self, computer_instance)
+```
### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `PAGE_DOWN` | `Any` | |
-| `PAGE_UP` | `Any` | |
-| `HOME` | `Any` | |
-| `END` | `Any` | |
-| `LEFT` | `Any` | |
-| `RIGHT` | `Any` | |
-| `UP` | `Any` | |
-| `DOWN` | `Any` | |
-| `RETURN` | `Any` | |
-| `ENTER` | `Any` | |
-| `ESCAPE` | `Any` | |
-| `ESC` | `Any` | |
-| `TAB` | `Any` | |
-| `SPACE` | `Any` | |
-| `BACKSPACE` | `Any` | |
-| `DELETE` | `Any` | |
-| `ALT` | `Any` | |
-| `CTRL` | `Any` | |
-| `SHIFT` | `Any` | |
-| `WIN` | `Any` | |
-| `COMMAND` | `Any` | |
-| `OPTION` | `Any` | |
-| `F1` | `Any` | |
-| `F2` | `Any` | |
-| `F3` | `Any` | |
-| `F4` | `Any` | |
-| `F5` | `Any` | |
-| `F6` | `Any` | |
-| `F7` | `Any` | |
-| `F8` | `Any` | |
-| `F9` | `Any` | |
-| `F10` | `Any` | |
-| `F11` | `Any` | |
-| `F12` | `Any` | |
+| `is_tracing` | `bool` | Check if tracing is currently active. |
### Methods
-#### Key.from_string
+#### ComputerTracing.start
```python
-def from_string(cls, key: str) -> Key | str
+async def start(self, config: Optional[Dict[str, Any]] = None) -> None
```
-Convert a string key name to a Key enum value.
+Start tracing with the specified configuration.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `key` | `Any` | String key name to convert |
+| `config` | `Any` | Tracing configuration dict with options: - video: bool - Record video frames (default: False) - screenshots: bool - Record screenshots (default: True) - api_calls: bool - Record API calls and results (default: True) - accessibility_tree: bool - Record accessibility tree snapshots (default: False) - metadata: bool - Record custom metadata (default: True) - name: str - Custom trace name (default: auto-generated) - path: str - Custom trace directory path (default: auto-generated) |
-**Returns:** Key enum value if the string matches a known key, otherwise returns the original string for single character keys
+#### ComputerTracing.stop
----
+```python
+async def stop(self, options: Optional[Dict[str, Any]] = None) -> str
+```
-## DioramaComputer
+Stop tracing and save the trace data.
-A Computer-compatible proxy for Diorama that sends commands over the ComputerInterface.
+**Parameters:**
-### Constructor
+| Name | Type | Description |
+|------|------|-------------|
+| `options` | `Any` | Stop options dict with: - path: str - Custom output path for the trace archive - format: str - Output format ('zip' or 'dir', default: 'zip') |
+
+**Returns:** str: Path to the saved trace file or directory
+
+#### ComputerTracing.record_api_call
```python
-DioramaComputer(self, computer, apps)
+async def record_api_call(self, method: str, args: Dict[str, Any], result: Any = None, error: Optional[Exception] = None) -> None
```
-### Attributes
+Record an API call event.
+
+**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `computer` | `Any` | |
-| `apps` | `Any` | |
-| `interface` | `Any` | |
+| `method` | `Any` | The method name that was called |
+| `args` | `Any` | Arguments passed to the method |
+| `result` | `Any` | Result returned by the method |
+| `error` | `Any` | Exception raised by the method, if any |
-### Methods
+#### ComputerTracing.record_accessibility_tree
-#### DioramaComputer.run
+```python
+async def record_accessibility_tree(self) -> None
+```
+
+Record the current accessibility tree if enabled.
+
+#### ComputerTracing.add_metadata
```python
-async def run(self)
+async def add_metadata(self, key: str, value: Any) -> None
```
-Initialize and run the DioramaComputer if not already initialized.
+Add custom metadata to the trace.
-**Returns:** self: The DioramaComputer instance
+**Parameters:**
+
+| Name | Type | Description |
+|------|------|-------------|
+| `key` | `Any` | Metadata key |
+| `value` | `Any` | Metadata value |
---
-## DioramaComputerInterface
+## models
-Diorama Interface proxy that sends diorama_cmds via the Computer's interface.
+Models for computer configuration.
-### Constructor
+---
-```python
-DioramaComputerInterface(self, computer, apps)
-```
+## BaseVMProvider
+
+*Inherits from: AsyncContextManager*
+
+Base interface for VM providers.
+
+All VM provider implementations must implement this interface.
### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `computer` | `Any` | |
-| `apps` | `Any` | |
+| `provider_type` | `VMProviderType` | Get the provider type. |
### Methods
-#### DioramaComputerInterface.screenshot
+#### BaseVMProvider.get_vm
```python
-async def screenshot(self, as_bytes = True)
+async def get_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]
```
-Take a screenshot of the diorama scene.
+Get VM information by name.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `as_bytes` | `bool` | If True, return image as bytes; if False, return PIL Image object |
+| `name` | `Any` | Name of the VM to get information for |
+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-**Returns:** bytes or PIL.Image: Screenshot data in the requested format
+**Returns:** Dictionary with VM information including status, IP address, etc.
-#### DioramaComputerInterface.get_screen_size
+#### BaseVMProvider.list_vms
```python
-async def get_screen_size(self)
+async def list_vms(self) -> ListVMsResponse
```
-Get the dimensions of the diorama scene.
+List all available VMs.
-**Returns:** dict: Dictionary containing 'width' and 'height' keys with pixel dimensions
+**Returns:** ListVMsResponse: A list of minimal VM objects as defined in `computer.providers.types.MinimalVM`.
-#### DioramaComputerInterface.move_cursor
+#### BaseVMProvider.run_vm
```python
-async def move_cursor(self, x, y)
+async def run_vm(self, image: str, name: str, run_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]
```
-Move the cursor to the specified coordinates.
+Run a VM by name with the given options.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `x` | `int` | X coordinate to move cursor to |
-| `y` | `int` | Y coordinate to move cursor to |
+| `image` | `Any` | Name/tag of the image to use |
+| `name` | `Any` | Name of the VM to run |
+| `run_opts` | `Any` | Dictionary of run options (memory, cpu, etc.) |
+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-#### DioramaComputerInterface.left_click
+**Returns:** Dictionary with VM run status and information
+
+#### BaseVMProvider.stop_vm
```python
-async def left_click(self, x = None, y = None)
+async def stop_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]
```
-Perform a left mouse click at the specified coordinates or current cursor position.
+Stop a VM by name.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |
-| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |
+| `name` | `Any` | Name of the VM to stop |
+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-#### DioramaComputerInterface.right_click
+**Returns:** Dictionary with VM stop status and information
+
+#### BaseVMProvider.restart_vm
```python
-async def right_click(self, x = None, y = None)
+async def restart_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]
```
-Perform a right mouse click at the specified coordinates or current cursor position.
+Restart a VM by name.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |
-| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |
+| `name` | `Any` | Name of the VM to restart |
+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-#### DioramaComputerInterface.double_click
+**Returns:** Dictionary with VM restart status and information
+
+#### BaseVMProvider.update_vm
```python
-async def double_click(self, x = None, y = None)
+async def update_vm(self, name: str, update_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]
```
-Perform a double mouse click at the specified coordinates or current cursor position.
+Update VM configuration.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `x` | `int, optional` | X coordinate to double-click at. If None, clicks at current cursor position |
-| `y` | `int, optional` | Y coordinate to double-click at. If None, clicks at current cursor position |
+| `name` | `Any` | Name of the VM to update |
+| `update_opts` | `Any` | Dictionary of update options (memory, cpu, etc.) |
+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-#### DioramaComputerInterface.scroll_up
+**Returns:** Dictionary with VM update status and information
+
+#### BaseVMProvider.get_ip
```python
-async def scroll_up(self, clicks = 1)
+async def get_ip(self, name: str, storage: Optional[str] = None, retry_delay: int = 2) -> str
```
-Scroll up by the specified number of clicks.
+Get the IP address of a VM, waiting indefinitely until it's available.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `clicks` | `int` | Number of scroll clicks to perform upward. Defaults to 1 |
-
-#### DioramaComputerInterface.scroll_down
+| `name` | `Any` | Name of the VM to get the IP for |
+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
+| `retry_delay` | `Any` | Delay between retries in seconds (default: 2) |
-```python
-async def scroll_down(self, clicks = 1)
-```
+**Returns:** IP address of the VM when it becomes available
-Scroll down by the specified number of clicks.
+---
-**Parameters:**
+## Display
-| Name | Type | Description |
-|------|------|-------------|
-| `clicks` | `int` | Number of scroll clicks to perform downward. Defaults to 1 |
+Display configuration.
-#### DioramaComputerInterface.drag_to
+### Constructor
```python
-async def drag_to(self, x, y, duration = 0.5)
+Display(self, width: int, height: int) -> None
```
-Drag from the current cursor position to the specified coordinates.
-
-**Parameters:**
+### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `x` | `int` | X coordinate to drag to |
-| `y` | `int` | Y coordinate to drag to |
-| `duration` | `float` | Duration of the drag operation in seconds. Defaults to 0.5 |
-
-#### DioramaComputerInterface.get_cursor_position
+| `width` | `int` | |
+| `height` | `int` | |
-```python
-async def get_cursor_position(self)
-```
+---
-Get the current cursor position.
+## Image
-**Returns:** dict: Dictionary containing the current cursor coordinates
+VM image configuration.
-#### DioramaComputerInterface.type_text
+### Constructor
```python
-async def type_text(self, text)
+Image(self, image: str, tag: str, name: str) -> None
```
-Type the specified text at the current cursor position.
-
-**Parameters:**
+### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `text` | `str` | The text to type |
-
-#### DioramaComputerInterface.press_key
-
-```python
-async def press_key(self, key)
-```
+| `image` | `str` | |
+| `tag` | `str` | |
+| `name` | `str` | |
-Press a single key.
+---
-**Parameters:**
+## Computer
-| Name | Type | Description |
-|------|------|-------------|
-| `key` | `Any` | The key to press |
+Computer configuration.
-#### DioramaComputerInterface.hotkey
+### Constructor
```python
-async def hotkey(self, keys = ())
+Computer(self, image: str, tag: str, name: str, display: Display, memory: str, cpu: str, vm_provider: Optional[BaseVMProvider] = None) -> None
```
-Press multiple keys simultaneously as a hotkey combination.
+### Attributes
-**Raises:**
+| Name | Type | Description |
+|------|------|-------------|
+| `image` | `str` | |
+| `tag` | `str` | |
+| `name` | `str` | |
+| `display` | `Display` | |
+| `memory` | `str` | |
+| `cpu` | `str` | |
+| `vm_provider` | `Optional[BaseVMProvider]` | |
-- `ValueError` - If any key is not a Key enum or string type
+### Methods
-#### DioramaComputerInterface.to_screen_coordinates
+#### Computer.get_ip
```python
-async def to_screen_coordinates(self, x, y)
+async def get_ip(self) -> Optional[str]
```
-Convert coordinates to screen coordinates.
-
-**Parameters:**
+Get the IP address of the VM.
-| Name | Type | Description |
-|------|------|-------------|
-| `x` | `int` | X coordinate to convert |
-| `y` | `int` | Y coordinate to convert |
+---
-**Returns:** dict: Dictionary containing the converted screen coordinates
+## diorama_computer
---
-## helpers
-
-Helper functions and decorators for the Computer module.
+## Key
----
+*Inherits from: Enum*
-## DependencyInfo
+Keyboard keys that can be used with press_key.
-*Inherits from: TypedDict*
+These key names follow a consistent cross-platform keyboard key naming convention.
### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `import_statements` | `List[str]` | |
-| `definitions` | `List[tuple[str, Any]]` | |
+| `PAGE_DOWN` | `Any` | |
+| `PAGE_UP` | `Any` | |
+| `HOME` | `Any` | |
+| `END` | `Any` | |
+| `LEFT` | `Any` | |
+| `RIGHT` | `Any` | |
+| `UP` | `Any` | |
+| `DOWN` | `Any` | |
+| `RETURN` | `Any` | |
+| `ENTER` | `Any` | |
+| `ESCAPE` | `Any` | |
+| `ESC` | `Any` | |
+| `TAB` | `Any` | |
+| `SPACE` | `Any` | |
+| `BACKSPACE` | `Any` | |
+| `DELETE` | `Any` | |
+| `ALT` | `Any` | |
+| `CTRL` | `Any` | |
+| `SHIFT` | `Any` | |
+| `WIN` | `Any` | |
+| `COMMAND` | `Any` | |
+| `OPTION` | `Any` | |
+| `F1` | `Any` | |
+| `F2` | `Any` | |
+| `F3` | `Any` | |
+| `F4` | `Any` | |
+| `F5` | `Any` | |
+| `F6` | `Any` | |
+| `F7` | `Any` | |
+| `F8` | `Any` | |
+| `F9` | `Any` | |
+| `F10` | `Any` | |
+| `F11` | `Any` | |
+| `F12` | `Any` | |
-### set_default_computer
+### Methods
+
+#### Key.from_string
```python
-def set_default_computer(computer: Any) -> None
+def from_string(cls, key: str) -> Key | str
```
-Set the default computer instance to be used by the remote decorator.
+Convert a string key name to a Key enum value.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `computer` | `Any` | The computer instance to use as default |
-
-### sandboxed
-
-```python
-def sandboxed(venv_name: str = 'default', computer: str = 'default', max_retries: int = 3) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]
-```
+| `key` | `Any` | String key name to convert |
-Decorator that wraps a function to be executed remotely via computer.venv_exec
+**Returns:** Key enum value if the string matches a known key, otherwise returns the original string for single character keys
-The function is automatically analyzed for dependencies (imports, helper functions,
-constants, etc.) and reconstructed with all necessary code in the remote sandbox.
+---
-**Parameters:**
+## DioramaComputer
-| Name | Type | Description |
-|------|------|-------------|
-| `venv_name` | `Any` | Name of the virtual environment to execute in |
-| `computer` | `Any` | The computer instance to use, or "default" to use the globally set default |
-| `max_retries` | `Any` | Maximum number of retries for the remote execution |
+A Computer-compatible proxy for Diorama that sends commands over the ComputerInterface.
-### generate_source_code
+### Constructor
```python
-def generate_source_code(func: FunctionType) -> str
+DioramaComputer(self, computer, apps)
```
-Generate complete source code for a function with all dependencies.
-
-**Parameters:**
+### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `func` | `Any` | The function to generate source code for |
+| `computer` | `Any` | |
+| `apps` | `Any` | |
+| `interface` | `Any` | |
-**Returns:** Complete Python source code as a string
+### Methods
----
+#### DioramaComputer.run
-## models
+```python
+async def run(self)
+```
-Models for computer configuration.
+Initialize and run the DioramaComputer if not already initialized.
+
+**Returns:** self: The DioramaComputer instance
---
-## BaseVMProvider
+## DioramaComputerInterface
-*Inherits from: AsyncContextManager*
+Diorama Interface proxy that sends diorama_cmds via the Computer's interface.
-Base interface for VM providers.
+### Constructor
-All VM provider implementations must implement this interface.
+```python
+DioramaComputerInterface(self, computer, apps)
+```
### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `provider_type` | `VMProviderType` | Get the provider type. |
+| `computer` | `Any` | |
+| `apps` | `Any` | |
### Methods
-#### BaseVMProvider.get_vm
+#### DioramaComputerInterface.screenshot
```python
-async def get_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]
+async def screenshot(self, as_bytes = True)
```
-Get VM information by name.
+Take a screenshot of the diorama scene.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `name` | `Any` | Name of the VM to get information for |
-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
+| `as_bytes` | `bool` | If True, return image as bytes; if False, return PIL Image object |
-**Returns:** Dictionary with VM information including status, IP address, etc.
+**Returns:** bytes or PIL.Image: Screenshot data in the requested format
-#### BaseVMProvider.list_vms
+#### DioramaComputerInterface.get_screen_size
```python
-async def list_vms(self) -> ListVMsResponse
+async def get_screen_size(self)
```
-List all available VMs.
+Get the dimensions of the diorama scene.
-**Returns:** ListVMsResponse: A list of minimal VM objects as defined in `computer.providers.types.MinimalVM`.
+**Returns:** dict: Dictionary containing 'width' and 'height' keys with pixel dimensions
-#### BaseVMProvider.run_vm
+#### DioramaComputerInterface.move_cursor
```python
-async def run_vm(self, image: str, name: str, run_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]
+async def move_cursor(self, x, y)
```
-Run a VM by name with the given options.
+Move the cursor to the specified coordinates.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `image` | `Any` | Name/tag of the image to use |
-| `name` | `Any` | Name of the VM to run |
-| `run_opts` | `Any` | Dictionary of run options (memory, cpu, etc.) |
-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-
-**Returns:** Dictionary with VM run status and information
+| `x` | `int` | X coordinate to move cursor to |
+| `y` | `int` | Y coordinate to move cursor to |
-#### BaseVMProvider.stop_vm
+#### DioramaComputerInterface.left_click
```python
-async def stop_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]
+async def left_click(self, x = None, y = None)
```
-Stop a VM by name.
+Perform a left mouse click at the specified coordinates or current cursor position.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `name` | `Any` | Name of the VM to stop |
-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-
-**Returns:** Dictionary with VM stop status and information
+| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |
+| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |
-#### BaseVMProvider.restart_vm
+#### DioramaComputerInterface.right_click
```python
-async def restart_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]
+async def right_click(self, x = None, y = None)
```
-Restart a VM by name.
+Perform a right mouse click at the specified coordinates or current cursor position.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `name` | `Any` | Name of the VM to restart |
-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-
-**Returns:** Dictionary with VM restart status and information
+| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |
+| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |
-#### BaseVMProvider.update_vm
+#### DioramaComputerInterface.double_click
```python
-async def update_vm(self, name: str, update_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]
+async def double_click(self, x = None, y = None)
```
-Update VM configuration.
+Perform a double mouse click at the specified coordinates or current cursor position.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `name` | `Any` | Name of the VM to update |
-| `update_opts` | `Any` | Dictionary of update options (memory, cpu, etc.) |
-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-
-**Returns:** Dictionary with VM update status and information
+| `x` | `int, optional` | X coordinate to double-click at. If None, clicks at current cursor position |
+| `y` | `int, optional` | Y coordinate to double-click at. If None, clicks at current cursor position |
-#### BaseVMProvider.get_ip
+#### DioramaComputerInterface.scroll_up
```python
-async def get_ip(self, name: str, storage: Optional[str] = None, retry_delay: int = 2) -> str
+async def scroll_up(self, clicks = 1)
```
-Get the IP address of a VM, waiting indefinitely until it's available.
+Scroll up by the specified number of clicks.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `name` | `Any` | Name of the VM to get the IP for |
-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |
-| `retry_delay` | `Any` | Delay between retries in seconds (default: 2) |
-
-**Returns:** IP address of the VM when it becomes available
-
----
-
-## Display
-
-Display configuration.
+| `clicks` | `int` | Number of scroll clicks to perform upward. Defaults to 1 |
-### Constructor
+#### DioramaComputerInterface.scroll_down
```python
-Display(self, width: int, height: int) -> None
+async def scroll_down(self, clicks = 1)
```
-### Attributes
+Scroll down by the specified number of clicks.
+
+**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `width` | `int` | |
-| `height` | `int` | |
-
----
-
-## Image
-
-VM image configuration.
+| `clicks` | `int` | Number of scroll clicks to perform downward. Defaults to 1 |
-### Constructor
+#### DioramaComputerInterface.drag_to
```python
-Image(self, image: str, tag: str, name: str) -> None
+async def drag_to(self, x, y, duration = 0.5)
```
-### Attributes
+Drag from the current cursor position to the specified coordinates.
+
+**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `image` | `str` | |
-| `tag` | `str` | |
-| `name` | `str` | |
+| `x` | `int` | X coordinate to drag to |
+| `y` | `int` | Y coordinate to drag to |
+| `duration` | `float` | Duration of the drag operation in seconds. Defaults to 0.5 |
----
+#### DioramaComputerInterface.get_cursor_position
-## Computer
+```python
+async def get_cursor_position(self)
+```
-Computer configuration.
+Get the current cursor position.
-### Constructor
+**Returns:** dict: Dictionary containing the current cursor coordinates
+
+#### DioramaComputerInterface.type_text
```python
-Computer(self, image: str, tag: str, name: str, display: Display, memory: str, cpu: str, vm_provider: Optional[BaseVMProvider] = None) -> None
+async def type_text(self, text)
```
-### Attributes
+Type the specified text at the current cursor position.
+
+**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `image` | `str` | |
-| `tag` | `str` | |
-| `name` | `str` | |
-| `display` | `Display` | |
-| `memory` | `str` | |
-| `cpu` | `str` | |
-| `vm_provider` | `Optional[BaseVMProvider]` | |
-
-### Methods
+| `text` | `str` | The text to type |
-#### Computer.get_ip
+#### DioramaComputerInterface.press_key
```python
-async def get_ip(self) -> Optional[str]
+async def press_key(self, key)
```
-Get the IP address of the VM.
-
----
+Press a single key.
-## tracing
+**Parameters:**
-Computer tracing functionality for recording sessions.
+| Name | Type | Description |
+|------|------|-------------|
+| `key` | `Any` | The key to press |
-This module provides a Computer.tracing API inspired by Playwright's tracing functionality,
-allowing users to record computer interactions for debugging, training, and analysis.
+#### DioramaComputerInterface.hotkey
----
+```python
+async def hotkey(self, keys = ())
+```
-## ComputerTracing
+Press multiple keys simultaneously as a hotkey combination.
-Computer tracing class that records computer interactions and saves them to disk.
+**Raises:**
-This class provides a flexible API for recording computer sessions with configurable
-options for what to record (screenshots, API calls, video, etc.).
+- `ValueError` - If any key is not a Key enum or string type
-### Constructor
+#### DioramaComputerInterface.to_screen_coordinates
```python
-ComputerTracing(self, computer_instance)
+async def to_screen_coordinates(self, x, y)
```
-### Attributes
+Convert coordinates to screen coordinates.
+
+**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `is_tracing` | `bool` | Check if tracing is currently active. |
+| `x` | `int` | X coordinate to convert |
+| `y` | `int` | Y coordinate to convert |
-### Methods
+**Returns:** dict: Dictionary containing the converted screen coordinates
-#### ComputerTracing.start
+---
-```python
-async def start(self, config: Optional[Dict[str, Any]] = None) -> None
-```
+## helpers
-Start tracing with the specified configuration.
+Helper functions and decorators for the Computer module.
-**Parameters:**
+---
+
+## DependencyInfo
+
+*Inherits from: TypedDict*
+
+### Attributes
| Name | Type | Description |
|------|------|-------------|
-| `config` | `Any` | Tracing configuration dict with options: - video: bool - Record video frames (default: False) - screenshots: bool - Record screenshots (default: True) - api_calls: bool - Record API calls and results (default: True) - accessibility_tree: bool - Record accessibility tree snapshots (default: False) - metadata: bool - Record custom metadata (default: True) - name: str - Custom trace name (default: auto-generated) - path: str - Custom trace directory path (default: auto-generated) |
+| `import_statements` | `List[str]` | |
+| `definitions` | `List[tuple[str, Any]]` | |
-#### ComputerTracing.stop
+### set_default_computer
```python
-async def stop(self, options: Optional[Dict[str, Any]] = None) -> str
+def set_default_computer(computer: Any) -> None
```
-Stop tracing and save the trace data.
+Set the default computer instance to be used by the remote decorator.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `options` | `Any` | Stop options dict with: - path: str - Custom output path for the trace archive - format: str - Output format ('zip' or 'dir', default: 'zip') |
-
-**Returns:** str: Path to the saved trace file or directory
+| `computer` | `Any` | The computer instance to use as default |
-#### ComputerTracing.record_api_call
+### sandboxed
```python
-async def record_api_call(self, method: str, args: Dict[str, Any], result: Any = None, error: Optional[Exception] = None) -> None
+def sandboxed(venv_name: str = 'default', computer: str = 'default', max_retries: int = 3) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]
```
-Record an API call event.
+Decorator that wraps a function to be executed remotely via computer.venv_exec
+
+The function is automatically analyzed for dependencies (imports, helper functions,
+constants, etc.) and reconstructed with all necessary code in the remote sandbox.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `method` | `Any` | The method name that was called |
-| `args` | `Any` | Arguments passed to the method |
-| `result` | `Any` | Result returned by the method |
-| `error` | `Any` | Exception raised by the method, if any |
-
-#### ComputerTracing.record_accessibility_tree
-
-```python
-async def record_accessibility_tree(self) -> None
-```
-
-Record the current accessibility tree if enabled.
+| `venv_name` | `Any` | Name of the virtual environment to execute in |
+| `computer` | `Any` | The computer instance to use, or "default" to use the globally set default |
+| `max_retries` | `Any` | Maximum number of retries for the remote execution |
-#### ComputerTracing.add_metadata
+### generate_source_code
```python
-async def add_metadata(self, key: str, value: Any) -> None
+def generate_source_code(func: FunctionType) -> str
```
-Add custom metadata to the trace.
+Generate complete source code for a function with all dependencies.
**Parameters:**
| Name | Type | Description |
|------|------|-------------|
-| `key` | `Any` | Metadata key |
-| `value` | `Any` | Metadata value |
+| `func` | `Any` | The function to generate source code for |
+
+**Returns:** Complete Python source code as a string
---
diff --git a/docs/content/docs/cuabot/reference/index.mdx b/docs/content/docs/cuabot/reference/index.mdx
index 90eacfd324..abe9868fd9 100644
--- a/docs/content/docs/cuabot/reference/index.mdx
+++ b/docs/content/docs/cuabot/reference/index.mdx
@@ -7,7 +7,7 @@ description: TypeScript API reference for the Cua-Bot sandboxed agent framework
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/typescript-sdk.ts
Source: libs/cuabot/src
- Version: 1.0.13
+ Version: 1.0.14
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -16,7 +16,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"1.0","href":"/cuabot/reference","isCurrent":true}]}
currentVersion="1.0"
- fullVersion="1.0.13"
+ fullVersion="1.0.14"
packageName="cuabot"
installCommand="npm install -g cuabot"
/>
diff --git a/docs/content/docs/lume/reference/cli-reference.mdx b/docs/content/docs/lume/reference/cli-reference.mdx
index a8c3427314..1ee83e3451 100644
--- a/docs/content/docs/lume/reference/cli-reference.mdx
+++ b/docs/content/docs/lume/reference/cli-reference.mdx
@@ -7,7 +7,7 @@ description: Command Line Interface reference for Lume
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/lume.ts
Source: libs/lume/src/Commands/*.swift
- Version: 0.2.76
+ Version: 0.2.81
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -16,7 +16,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}
currentVersion="0.2"
- fullVersion="0.2.76"
+ fullVersion="0.2.81"
packageName="lume"
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"
/>
diff --git a/docs/content/docs/lume/reference/http-api.mdx b/docs/content/docs/lume/reference/http-api.mdx
index 524738783a..b205e76e9d 100644
--- a/docs/content/docs/lume/reference/http-api.mdx
+++ b/docs/content/docs/lume/reference/http-api.mdx
@@ -7,7 +7,7 @@ description: HTTP API reference for Lume server
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/lume.ts
Source: libs/lume/src/Server/*.swift
- Version: 0.2.76
+ Version: 0.2.81
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}
currentVersion="0.2"
- fullVersion="0.2.76"
+ fullVersion="0.2.81"
packageName="lume"
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"
/>
@@ -392,6 +392,7 @@ Start a virtual machine
| sharedDirectories | array | No | Directories to share with the VM |
| recoveryMode | boolean | No | Boot macOS VM in recovery mode (default: false) |
| storage | string | No | VM storage location |
+| clipboard | boolean | No | Enable bidirectional clipboard sync via SSH (experimental) (default: false) |
#### Example Request
diff --git a/libs/python/agent/agent/decorators.py b/libs/python/agent/agent/decorators.py
index 4c588a2660..c662221f38 100644
--- a/libs/python/agent/agent/decorators.py
+++ b/libs/python/agent/agent/decorators.py
@@ -60,9 +60,34 @@ def get_agent_configs() -> List[AgentConfigInfo]:
return _agent_configs.copy()
+def _strip_cua_prefix(model: str) -> str:
+ """Strip the ``cua/<provider>/`` routing prefix so the bare model name
+ can be matched against registered agent patterns.
+
+ Examples:
+ cua/google/gemini-3-flash-preview -> gemini-3-flash-preview
+ cua/anthropic/claude-sonnet-4-6 -> claude-sonnet-4-6
+ gemini-3-flash-preview -> gemini-3-flash-preview (unchanged)
+ """
+ parts = model.split("/")
+ if parts[0] == "cua" and len(parts) >= 3:
+ return "/".join(parts[2:])
+ return model
+
+
def find_agent_config(model: str) -> Optional[AgentConfigInfo]:
- """Find the best matching agent config for a model"""
+ """Find the best matching agent config for a model.
+
+ For each registered config (checked in priority order), tries the
+ original model string first and then the bare model name with the
+ ``cua/<provider>/`` routing prefix stripped. This ensures that
+ routed models (e.g. ``cua/google/gemini-3-flash-preview``) resolve
+ to the same agent loop as their bare counterparts.
+ """
+ stripped = _strip_cua_prefix(model)
for config_info in _agent_configs:
if config_info.matches_model(model):
return config_info
+ if stripped != model and config_info.matches_model(stripped):
+ return config_info
return None
diff --git a/libs/python/agent/agent/loops/gemini.py b/libs/python/agent/agent/loops/gemini.py
index e2411d8bfe..86221d779a 100644
--- a/libs/python/agent/agent/loops/gemini.py
+++ b/libs/python/agent/agent/loops/gemini.py
@@ -27,6 +27,7 @@ from PIL import Image
from ..decorators import register_agent
from ..loops.base import AsyncAgentConfig
+from ..responses import make_reasoning_item
from ..types import AgentCapability
@@ -92,6 +93,58 @@ def _sanitize_for_json(obj: Any) -> Any:
return str(obj)
+def _create_gemini_client(
+ original_model: str, genai: Any, kwargs: Dict[str, Any]
+) -> Tuple[Any, str]:
+ """Create a Gemini SDK client, routing through CUA proxy if model has cua/ prefix.
+
+ Returns (client, bare_model_name).
+
+ When the model string starts with ``cua/<provider>/`` the Google GenAI SDK
+ is configured to send requests through the CUA inference proxy at
+ ``{CUA_BASE_URL}/gemini``. This keeps the Gemini loop as the single code
+ path for both direct-Google and CUA-routed Gemini models.
+ """
+ import os
+
+ from ..decorators import _strip_cua_prefix
+
+ model = _strip_cua_prefix(original_model)
+ is_cua_routed = original_model != model
+
+ if is_cua_routed:
+ api_key = (
+ kwargs.get("api_key") or os.getenv("CUA_INFERENCE_API_KEY") or os.getenv("CUA_API_KEY")
+ )
+ if not api_key:
+ raise ValueError(
+ "No CUA API key found for cua/ model routing. "
+ "Set CUA_API_KEY environment variable or pass api_key to ComputerAgent()."
+ )
+ cua_base_url = os.getenv("CUA_BASE_URL", "https://inference.cua.ai/v1")
+ http_options: Dict[str, Any] = {"base_url": f"{cua_base_url}/gemini"}
+ # Include CUA version headers if available
+ try:
+ from core.http import cua_version_headers
+
+ hdrs = cua_version_headers()
+ if hdrs:
+ http_options["headers"] = hdrs
+ except Exception:
+ pass
+ client = genai.Client(api_key=api_key, http_options=http_options)
+ else:
+ api_key = kwargs.get("api_key", os.getenv("GOOGLE_API_KEY"))
+ if api_key:
+ client = genai.Client(api_key=api_key)
+ else:
+ # Vertex AI mode - requires GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION env vars
+ # and Application Default Credentials (ADC)
+ client = genai.Client()
+
+ return client, model
+
+
def _find_last_user_text(messages: List[Dict[str, Any]]) -> List[str]:
texts: List[str] = []
for msg in reversed(messages):
@@ -219,7 +272,8 @@ def _convert_messages_to_gemini_contents(
types.Content(
role="user",
parts=[
- types.Part.from_bytes(data=img_bytes, mime_type="image/png")
+ types.Part(text="[screenshot]"),
+ types.Part.from_bytes(data=img_bytes, mime_type="image/png"),
],
)
)
@@ -303,24 +357,36 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:
return [
types.FunctionDeclaration(
name="click_at",
- description="Click at the specified x,y coordinates on the screen. Coordinates are normalized 0-999.",
+ description="Click at the specified x,y coordinates on the screen. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen. Look carefully at the screenshot to identify the exact position of the target element before clicking.",
parameters={
"type": "object",
"properties": {
- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},
- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},
+ "x": {
+ "type": "integer",
+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
+ },
+ "y": {
+ "type": "integer",
+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
+ },
},
"required": ["x", "y"],
},
),
types.FunctionDeclaration(
name="type_text_at",
- description="Type text at the specified x,y coordinates. First clicks at the location, then types the text.",
+ description="Type text at the specified x,y coordinates. First clicks at the location, then types the text. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",
parameters={
"type": "object",
"properties": {
- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},
- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},
+ "x": {
+ "type": "integer",
+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
+ },
+ "y": {
+ "type": "integer",
+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
+ },
"text": {"type": "string", "description": "Text to type"},
"press_enter": {
"type": "boolean",
@@ -332,12 +398,18 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:
),
types.FunctionDeclaration(
name="hover_at",
- description="Move the mouse cursor to the specified x,y coordinates without clicking.",
+ description="Move the mouse cursor to the specified x,y coordinates without clicking. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",
parameters={
"type": "object",
"properties": {
- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},
- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},
+ "x": {
+ "type": "integer",
+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
+ },
+ "y": {
+ "type": "integer",
+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
+ },
},
"required": ["x", "y"],
},
@@ -358,12 +430,18 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:
),
types.FunctionDeclaration(
name="scroll_at",
- description="Scroll at the specified x,y coordinates in a given direction.",
+ description="Scroll at the specified x,y coordinates in a given direction. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",
parameters={
"type": "object",
"properties": {
- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},
- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},
+ "x": {
+ "type": "integer",
+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
+ },
+ "y": {
+ "type": "integer",
+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
+ },
"direction": {
"type": "string",
"enum": ["up", "down", "left", "right"],
@@ -394,25 +472,25 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:
),
types.FunctionDeclaration(
name="drag_and_drop",
- description="Drag from one coordinate to another.",
+ description="Drag from one coordinate to another. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",
parameters={
"type": "object",
"properties": {
"x": {
"type": "integer",
- "description": "Starting X coordinate (0-999 normalized)",
+ "description": "Starting X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
},
"y": {
"type": "integer",
- "description": "Starting Y coordinate (0-999 normalized)",
+ "description": "Starting Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
},
"destination_x": {
"type": "integer",
- "description": "Destination X coordinate (0-999 normalized)",
+ "description": "Destination X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
},
"destination_y": {
"type": "integer",
- "description": "Destination Y coordinate (0-999 normalized)",
+ "description": "Destination Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
},
},
"required": ["x", "y", "destination_x", "destination_y"],
@@ -625,19 +703,9 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
**kwargs,
) -> Dict[str, Any]:
genai, types = _lazy_import_genai()
- import os
- # Authentication follows two modes based on environment variables:
- # 1. Google AI Studio: Set GOOGLE_API_KEY
- # 2. Vertex AI: Set GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION, GOOGLE_GENAI_USE_VERTEXAI=True
- api_key = kwargs.get("api_key", os.getenv("GOOGLE_API_KEY"))
-
- if api_key:
- client = genai.Client(api_key=api_key)
- else:
- # Vertex AI mode - requires GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION env vars
- # and Application Default Credentials (ADC)
- client = genai.Client()
+ # Create client with CUA routing support (detects cua/ prefix automatically)
+ client, model = _create_gemini_client(model, genai, kwargs)
# Extract Gemini 3-specific parameters
# thinking_level: Use types.ThinkingLevel enum values (e.g., "LOW", "HIGH", "MEDIUM", "MINIMAL")
@@ -678,6 +746,10 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
# Assume it's already an SDK enum value
resolved_media_resolution = media_resolution
+ # Convert full message history to Gemini Contents format
+ # (done early so screen dimensions are available for system instruction)
+ contents, (screen_w, screen_h) = _convert_messages_to_gemini_contents(messages, types)
+
# Compose tools config based on model type
# Gemini 2.5 Computer Use Preview uses built-in ComputerUse tool
# Gemini 3 Flash/Pro Preview uses custom function declarations
@@ -689,7 +761,19 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
print(f"[DEBUG] Using custom function declarations for Gemini 3 model: {model}")
print(f"[DEBUG] Number of custom functions: {len(custom_functions)}")
+ # Build system instruction with coordinate system and screen resolution context
+ # to improve coordinate precision for Gemini 3 models
+ system_instruction = (
+ f"You are controlling a computer with screen resolution {screen_w}x{screen_h} pixels. "
+ "When using coordinate-based functions (click_at, type_text_at, hover_at, scroll_at, drag_and_drop), "
+ "provide x and y as normalized values in the 0-999 range: "
+ "x=0 is the left edge, x=999 is the right edge; "
+ "y=0 is the top edge, y=999 is the bottom edge. "
+ "Look carefully at the screenshot to identify the exact position of UI elements before clicking."
+ )
+
generate_content_config = types.GenerateContentConfig(
+ system_instruction=system_instruction,
tools=[
types.Tool(function_declarations=custom_functions),
],
@@ -734,9 +818,6 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
media_resolution=resolved_media_resolution,
)
- # Convert full message history to Gemini Contents format
- contents, (screen_w, screen_h) = _convert_messages_to_gemini_contents(messages, types)
-
api_kwargs = {
"model": model,
"contents": contents,
@@ -800,6 +881,10 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
text_parts: List[str] = []
function_calls: List[Dict[str, Any]] = []
for p in candidate.content.parts:
+ # Check for thinking/reasoning content first
+ if getattr(p, "thought", False) and getattr(p, "text", None):
+ output_items.append(make_reasoning_item(p.text))
+ continue
if getattr(p, "text", None):
text_parts.append(p.text)
if getattr(p, "function_call", None):
@@ -844,14 +929,9 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
Returns pixel (x, y) if a click is proposed, else None.
"""
genai, types = _lazy_import_genai()
- import os
- # Authentication: GOOGLE_API_KEY for AI Studio, or Vertex AI env vars
- api_key = kwargs.get("api_key", os.getenv("GOOGLE_API_KEY"))
- if api_key:
- client = genai.Client(api_key=api_key)
- else:
- client = genai.Client()
+ # Create client with CUA routing support (detects cua/ prefix automatically)
+ client, model = _create_gemini_client(model, genai, kwargs)
# Build tools config based on model type
is_gemini_3 = _is_gemini_3_model(model)
@@ -860,12 +940,18 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
# For Gemini 3 models, use only click_at function declaration
click_function = types.FunctionDeclaration(
name="click_at",
- description="Click at the specified x,y coordinates on the screen. Coordinates are normalized 0-999.",
+ description="Click at the specified x,y coordinates on the screen. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen. Look carefully at the screenshot to identify the exact position of the target element before clicking.",
parameters={
"type": "object",
"properties": {
- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},
- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},
+ "x": {
+ "type": "integer",
+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",
+ },
+ "y": {
+ "type": "integer",
+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",
+ },
},
"required": ["x", "y"],
},
diff --git a/uv.lock b/uv.lock
index 1abfbee7de..8064b7ce49 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1039,7 +1039,7 @@ wheels = [
[[package]]
name = "cua-agent"
-version = "0.7.27"
+version = "0.7.29"
source = { editable = "libs/python/agent" }
dependencies = [
{ name = "aiohttp" },
@@ -1189,6 +1189,33 @@ requires-dist = [
]
provides-extras = ["openai", "anthropic", "qwen", "omni", "uitars", "uitars-mlx", "uitars-hf", "glm45v-hf", "opencua-hf", "internvl-hf", "moondream3", "ui", "cli", "hud", "gemini", "all"]
+[[package]]
+name = "cua-auto"
+version = "0.1.1"
+source = { editable = "libs/python/cua-auto" }
+dependencies = [
+ { name = "pillow" },
+ { name = "pynput" },
+ { name = "pyperclip" },
+ { name = "pywinctl" },
+]
+
+[package.metadata]
+requires-dist = [
+ { name = "cua-auto", extras = ["mss", "windows", "pty"], marker = "extra == 'all'" },
+ { name = "mss", marker = "extra == 'mss'", specifier = ">=9.0.0" },
+ { name = "pillow", specifier = ">=10.0.0" },
+ { name = "pynput", specifier = ">=1.7.0" },
+ { name = "pyperclip", specifier = ">=1.9.0" },
+ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
+ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" },
+ { name = "pywin32", marker = "sys_platform == 'win32' and extra == 'windows'", specifier = ">=306" },
+ { name = "pywinctl", specifier = ">=0.4" },
+ { name = "pywinpty", marker = "sys_platform == 'win32' and extra == 'pty'", specifier = ">=2.0.0" },
+ { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" },
+]
+provides-extras = ["mss", "windows", "pty", "all", "dev"]
+
[[package]]
name = "cua-bench-ui"
version = "0.7.0"
@@ -1208,7 +1235,7 @@ requires-dist = [
[[package]]
name = "cua-computer"
-version = "0.5.14"
+version = "0.5.16"
source = { editable = "libs/python/computer" }
dependencies = [
{ name = "aiohttp" },
@@ -1252,10 +1279,11 @@ provides-extras = ["lume", "lumier", "ui", "all"]
[[package]]
name = "cua-computer-server"
-version = "0.3.18"
+version = "0.3.19"
source = { editable = "libs/python/computer-server" }
dependencies = [
{ name = "aiohttp" },
+ { name = "cua-auto" },
{ name = "cua-core" },
{ name = "fastapi" },
{ name = "fastmcp" },
@@ -1291,6 +1319,7 @@ windows = [
[package.metadata]
requires-dist = [
{ name = "aiohttp", specifier = ">=3.9.1" },
+ { name = "cua-auto", editable = "libs/python/cua-auto" },
{ name = "cua-core", editable = "libs/python/core" },
{ name = "fastapi", specifier = ">=0.111.0" },
{ name = "fastmcp", specifier = ">=2.0,<3" },
@@ -1318,7 +1347,7 @@ provides-extras = ["macos", "linux", "windows"]
[[package]]
name = "cua-core"
-version = "0.1.17"
+version = "0.1.18"
source = { editable = "libs/python/core" }
dependencies = [
{ name = "posthog" },