Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Add window frame cmds

Dillon DuPont4823256fde01
diff --git a/docs/content/docs/computer-sdk/commands.mdx b/docs/content/docs/computer-sdk/commands.mdx
index 349d67d8fe..96330f396c 100644
--- a/docs/content/docs/computer-sdk/commands.mdx
+++ b/docs/content/docs/computer-sdk/commands.mdx
@@ -42,6 +42,11 @@ Control application launching and windows:
window_id = await computer.interface.get_current_window_id() # get the current active window id
await computer.interface.window_size(window_id)
await computer.interface.get_window_title(window_id)
+ await computer.interface.get_window_position(window_id)
+ await computer.interface.set_window_size(window_id, 1200, 800)
+ await computer.interface.set_window_position(window_id, 100, 100)
+ await computer.interface.maximize_window(window_id)
+ await computer.interface.minimize_window(window_id)
await computer.interface.close_window(window_id)
```
@@ -61,6 +66,11 @@ Control application launching and windows:
windowId = await computer.interface.getCurrentWindowId(); // current active window id
await computer.interface.getWindowSize(windowId);
await computer.interface.getWindowName(windowId);
+ await computer.interface.getWindowPosition(windowId);
+ await computer.interface.setWindowSize(windowId, 1200, 800);
+ await computer.interface.setWindowPosition(windowId, 100, 100);
+ await computer.interface.maximizeWindow(windowId);
+ await computer.interface.minimizeWindow(windowId);
await computer.interface.closeWindow(windowId);
```
diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.py
index a49f232cfc..8b2b7ac4e6 100644
--- a/libs/python/computer-server/computer_server/handlers/base.py
+++ b/libs/python/computer-server/computer_server/handlers/base.py
@@ -139,20 +139,45 @@ class BaseWindowHandler(ABC):
pass
@abstractmethod
- async def get_window_size(self, window_id: str) -> Dict[str, Any]:
+ async def get_window_size(self, window_id: str | int) -> Dict[str, Any]:
"""Get the size of a window by ID as {width, height}."""
pass
@abstractmethod
- async def activate_window(self, window_id: str) -> Dict[str, Any]:
+ async def activate_window(self, window_id: str | int) -> Dict[str, Any]:
"""Bring a window to the foreground by ID."""
pass
@abstractmethod
- async def close_window(self, window_id: str) -> Dict[str, Any]:
+ async def close_window(self, window_id: str | int) -> Dict[str, Any]:
"""Close a window by ID."""
pass
+ @abstractmethod
+ async def get_window_position(self, window_id: str | int) -> Dict[str, Any]:
+ """Get the top-left position of a window as {x, y}."""
+ pass
+
+ @abstractmethod
+ async def set_window_size(self, window_id: str | int, width: int, height: int) -> Dict[str, Any]:
+ """Set the size of a window by ID."""
+ pass
+
+ @abstractmethod
+ async def set_window_position(self, window_id: str | int, x: int, y: int) -> Dict[str, Any]:
+ """Set the position of a window by ID."""
+ pass
+
+ @abstractmethod
+ async def maximize_window(self, window_id: str | int) -> Dict[str, Any]:
+ """Maximize a window by ID."""
+ pass
+
+ @abstractmethod
+ async def minimize_window(self, window_id: str | int) -> Dict[str, Any]:
+ """Minimize a window by ID."""
+ pass
+
class BaseAutomationHandler(ABC):
"""Abstract base class for OS-specific automation handlers.
diff --git a/libs/python/computer-server/computer_server/handlers/generic.py b/libs/python/computer-server/computer_server/handlers/generic.py
index e39a693c2a..1c5509d48c 100644
--- a/libs/python/computer-server/computer_server/handlers/generic.py
+++ b/libs/python/computer-server/computer_server/handlers/generic.py
@@ -170,6 +170,66 @@ class GenericWindowHandler(BaseWindowHandler):
except Exception as e:
return {"success": False, "error": str(e)}
+ async def get_window_position(self, window_id: int | str) -> Dict[str, Any]:
+ try:
+ if pwc is None:
+ return {"success": False, "error": "pywinctl not available"}
+ w = self._get_window_by_id(window_id)
+ if not w:
+ return {"success": False, "error": "Window not found"}
+ x, y = w.position
+ return {"success": True, "x": int(x), "y": int(y)}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def set_window_size(self, window_id: int | str, width: int, height: int) -> Dict[str, Any]:
+ try:
+ if pwc is None:
+ return {"success": False, "error": "pywinctl not available"}
+ w = self._get_window_by_id(window_id)
+ if not w:
+ return {"success": False, "error": "Window not found"}
+ ok = w.resizeTo(int(width), int(height))
+ return {"success": bool(ok)}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def set_window_position(self, window_id: int | str, x: int, y: int) -> Dict[str, Any]:
+ try:
+ if pwc is None:
+ return {"success": False, "error": "pywinctl not available"}
+ w = self._get_window_by_id(window_id)
+ if not w:
+ return {"success": False, "error": "Window not found"}
+ ok = w.moveTo(int(x), int(y))
+ return {"success": bool(ok)}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def maximize_window(self, window_id: int | str) -> Dict[str, Any]:
+ try:
+ if pwc is None:
+ return {"success": False, "error": "pywinctl not available"}
+ w = self._get_window_by_id(window_id)
+ if not w:
+ return {"success": False, "error": "Window not found"}
+ ok = w.maximize()
+ return {"success": bool(ok)}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def minimize_window(self, window_id: int | str) -> Dict[str, Any]:
+ try:
+ if pwc is None:
+ return {"success": False, "error": "pywinctl not available"}
+ w = self._get_window_by_id(window_id)
+ if not w:
+ return {"success": False, "error": "Window not found"}
+ ok = w.minimize()
+ return {"success": bool(ok)}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
async def activate_window(self, window_id: int | str) -> Dict[str, Any]:
try:
if pwc is None:
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py
index fc9b6354f8..5a299ac42f 100644
--- a/libs/python/computer-server/computer_server/main.py
+++ b/libs/python/computer-server/computer_server/main.py
@@ -109,6 +109,11 @@ handlers = {
"get_application_windows": window_handler.get_application_windows,
"get_window_name": window_handler.get_window_name,
"get_window_size": window_handler.get_window_size,
+ "get_window_position": window_handler.get_window_position,
+ "set_window_size": window_handler.set_window_size,
+ "set_window_position": window_handler.set_window_position,
+ "maximize_window": window_handler.maximize_window,
+ "minimize_window": window_handler.minimize_window,
"activate_window": window_handler.activate_window,
"close_window": window_handler.close_window,
# Mouse commands
diff --git a/libs/python/computer/computer/interface/generic.py b/libs/python/computer/computer/interface/generic.py
index 80b2419942..e58719dde2 100644
--- a/libs/python/computer/computer/interface/generic.py
+++ b/libs/python/computer/computer/interface/generic.py
@@ -538,6 +538,36 @@ class GenericComputerInterface(BaseComputerInterface):
raise RuntimeError(result.get("error", "Failed to get window size"))
return int(result.get("width", 0)), int(result.get("height", 0))
+ async def get_window_position(self, window_id: int | str) -> tuple[int, int]:
+ result = await self._send_command("get_window_position", {"window_id": window_id})
+ if not result.get("success", False):
+ raise RuntimeError(result.get("error", "Failed to get window position"))
+ return int(result.get("x", 0)), int(result.get("y", 0))
+
+ async def set_window_size(self, window_id: int | str, width: int, height: int) -> None:
+ result = await self._send_command(
+ "set_window_size", {"window_id": window_id, "width": width, "height": height}
+ )
+ if not result.get("success", False):
+ raise RuntimeError(result.get("error", "Failed to set window size"))
+
+ async def set_window_position(self, window_id: int | str, x: int, y: int) -> None:
+ result = await self._send_command(
+ "set_window_position", {"window_id": window_id, "x": x, "y": y}
+ )
+ if not result.get("success", False):
+ raise RuntimeError(result.get("error", "Failed to set window position"))
+
+ async def maximize_window(self, window_id: int | str) -> None:
+ result = await self._send_command("maximize_window", {"window_id": window_id})
+ if not result.get("success", False):
+ raise RuntimeError(result.get("error", "Failed to maximize window"))
+
+ async def minimize_window(self, window_id: int | str) -> None:
+ result = await self._send_command("minimize_window", {"window_id": window_id})
+ if not result.get("success", False):
+ raise RuntimeError(result.get("error", "Failed to minimize window"))
+
async def activate_window(self, window_id: int | str) -> None:
result = await self._send_command("activate_window", {"window_id": window_id})
if not result.get("success", False):
diff --git a/libs/typescript/computer/src/interface/base.ts b/libs/typescript/computer/src/interface/base.ts
index 333060faf1..fe40442100 100644
--- a/libs/typescript/computer/src/interface/base.ts
+++ b/libs/typescript/computer/src/interface/base.ts
@@ -321,6 +321,11 @@ export abstract class BaseComputerInterface {
abstract getApplicationWindows(app: string): Promise<Array<number | string>>;
abstract getWindowName(windowId: number | string): Promise<string>;
abstract getWindowSize(windowId: number | string): Promise<[number, number]>;
+ abstract getWindowPosition(windowId: number | string): Promise<[number, number]>;
+ abstract setWindowSize(windowId: number | string, width: number, height: number): Promise<void>;
+ abstract setWindowPosition(windowId: number | string, x: number, y: number): Promise<void>;
+ abstract maximizeWindow(windowId: number | string): Promise<void>;
+ abstract minimizeWindow(windowId: number | string): Promise<void>;
abstract activateWindow(windowId: number | string): Promise<void>;
abstract closeWindow(windowId: number | string): Promise<void>;
diff --git a/libs/typescript/computer/src/interface/macos.ts b/libs/typescript/computer/src/interface/macos.ts
index 8b31768754..a113197fdf 100644
--- a/libs/typescript/computer/src/interface/macos.ts
+++ b/libs/typescript/computer/src/interface/macos.ts
@@ -266,6 +266,55 @@ export class MacOSComputerInterface extends BaseComputerInterface {
return [Number(response.width) || 0, Number(response.height) || 0];
}
+ /** Get window position as [x, y]. */
+ async getWindowPosition(windowId: number | string): Promise<[number, number]> {
+ const response = await this.sendCommand('get_window_position', { window_id: windowId });
+ if (!response.success) {
+ throw new Error((response.error as string) || 'Failed to get window position');
+ }
+ return [Number(response.x) || 0, Number(response.y) || 0];
+ }
+
+ /** Set window size. */
+ async setWindowSize(windowId: number | string, width: number, height: number): Promise<void> {
+ const response = await this.sendCommand('set_window_size', {
+ window_id: windowId,
+ width,
+ height,
+ });
+ if (!response.success) {
+ throw new Error((response.error as string) || 'Failed to set window size');
+ }
+ }
+
+ /** Set window position. */
+ async setWindowPosition(windowId: number | string, x: number, y: number): Promise<void> {
+ const response = await this.sendCommand('set_window_position', {
+ window_id: windowId,
+ x,
+ y,
+ });
+ if (!response.success) {
+ throw new Error((response.error as string) || 'Failed to set window position');
+ }
+ }
+
+ /** Maximize a window. */
+ async maximizeWindow(windowId: number | string): Promise<void> {
+ const response = await this.sendCommand('maximize_window', { window_id: windowId });
+ if (!response.success) {
+ throw new Error((response.error as string) || 'Failed to maximize window');
+ }
+ }
+
+ /** Minimize a window. */
+ async minimizeWindow(windowId: number | string): Promise<void> {
+ const response = await this.sendCommand('minimize_window', { window_id: windowId });
+ if (!response.success) {
+ throw new Error((response.error as string) || 'Failed to minimize window');
+ }
+ }
+
/** Activate a window by id. */
async activateWindow(windowId: number | string): Promise<void> {
const response = await this.sendCommand('activate_window', { window_id: windowId });