Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Add window frame cmds
Dillon DuPont4823256fde01
diff --git a/docs/content/docs/computer-sdk/commands.mdx b/docs/content/docs/computer-sdk/commands.mdxindex 349d67d8fe..96330f396c 100644--- a/docs/content/docs/computer-sdk/commands.mdx+++ b/docs/content/docs/computer-sdk/commands.mdx@@ -42,6 +42,11 @@ Control application launching and windows:window_id = await computer.interface.get_current_window_id() # get the current active window idawait computer.interface.window_size(window_id)await computer.interface.get_window_title(window_id)+ await computer.interface.get_window_position(window_id)+ await computer.interface.set_window_size(window_id, 1200, 800)+ await computer.interface.set_window_position(window_id, 100, 100)+ await computer.interface.maximize_window(window_id)+ await computer.interface.minimize_window(window_id)await computer.interface.close_window(window_id)```@@ -61,6 +66,11 @@ Control application launching and windows:windowId = await computer.interface.getCurrentWindowId(); // current active window idawait computer.interface.getWindowSize(windowId);await computer.interface.getWindowName(windowId);+ await computer.interface.getWindowPosition(windowId);+ await computer.interface.setWindowSize(windowId, 1200, 800);+ await computer.interface.setWindowPosition(windowId, 100, 100);+ await computer.interface.maximizeWindow(windowId);+ await computer.interface.minimizeWindow(windowId);await computer.interface.closeWindow(windowId);```diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.pyindex a49f232cfc..8b2b7ac4e6 100644--- a/libs/python/computer-server/computer_server/handlers/base.py+++ b/libs/python/computer-server/computer_server/handlers/base.py@@ -139,20 +139,45 @@ class BaseWindowHandler(ABC):pass@abstractmethod- async def get_window_size(self, window_id: str) -> Dict[str, Any]:+ async def get_window_size(self, window_id: str | int) -> Dict[str, Any]:"""Get the size of a window by ID as {width, height}."""pass@abstractmethod- async def activate_window(self, window_id: str) -> Dict[str, Any]:+ async def activate_window(self, window_id: str | int) -> Dict[str, Any]:"""Bring a window to the foreground by ID."""pass@abstractmethod- async def close_window(self, window_id: str) -> Dict[str, Any]:+ async def close_window(self, window_id: str | int) -> Dict[str, Any]:"""Close a window by ID."""pass+ @abstractmethod+ async def get_window_position(self, window_id: str | int) -> Dict[str, Any]:+ """Get the top-left position of a window as {x, y}."""+ pass++ @abstractmethod+ async def set_window_size(self, window_id: str | int, width: int, height: int) -> Dict[str, Any]:+ """Set the size of a window by ID."""+ pass++ @abstractmethod+ async def set_window_position(self, window_id: str | int, x: int, y: int) -> Dict[str, Any]:+ """Set the position of a window by ID."""+ pass++ @abstractmethod+ async def maximize_window(self, window_id: str | int) -> Dict[str, Any]:+ """Maximize a window by ID."""+ pass++ @abstractmethod+ async def minimize_window(self, window_id: str | int) -> Dict[str, Any]:+ """Minimize a window by ID."""+ pass+class BaseAutomationHandler(ABC):"""Abstract base class for OS-specific automation handlers.diff --git a/libs/python/computer-server/computer_server/handlers/generic.py b/libs/python/computer-server/computer_server/handlers/generic.pyindex e39a693c2a..1c5509d48c 100644--- a/libs/python/computer-server/computer_server/handlers/generic.py+++ b/libs/python/computer-server/computer_server/handlers/generic.py@@ -170,6 +170,66 @@ class GenericWindowHandler(BaseWindowHandler):except Exception as e:return {"success": False, "error": str(e)}+ async def get_window_position(self, window_id: int | str) -> Dict[str, Any]:+ try:+ if pwc is None:+ return {"success": False, "error": "pywinctl not available"}+ w = self._get_window_by_id(window_id)+ if not w:+ return {"success": False, "error": "Window not found"}+ x, y = w.position+ return {"success": True, "x": int(x), "y": int(y)}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def set_window_size(self, window_id: int | str, width: int, height: int) -> Dict[str, Any]:+ try:+ if pwc is None:+ return {"success": False, "error": "pywinctl not available"}+ w = self._get_window_by_id(window_id)+ if not w:+ return {"success": False, "error": "Window not found"}+ ok = w.resizeTo(int(width), int(height))+ return {"success": bool(ok)}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def set_window_position(self, window_id: int | str, x: int, y: int) -> Dict[str, Any]:+ try:+ if pwc is None:+ return {"success": False, "error": "pywinctl not available"}+ w = self._get_window_by_id(window_id)+ if not w:+ return {"success": False, "error": "Window not found"}+ ok = w.moveTo(int(x), int(y))+ return {"success": bool(ok)}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def maximize_window(self, window_id: int | str) -> Dict[str, Any]:+ try:+ if pwc is None:+ return {"success": False, "error": "pywinctl not available"}+ w = self._get_window_by_id(window_id)+ if not w:+ return {"success": False, "error": "Window not found"}+ ok = w.maximize()+ return {"success": bool(ok)}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def minimize_window(self, window_id: int | str) -> Dict[str, Any]:+ try:+ if pwc is None:+ return {"success": False, "error": "pywinctl not available"}+ w = self._get_window_by_id(window_id)+ if not w:+ return {"success": False, "error": "Window not found"}+ ok = w.minimize()+ return {"success": bool(ok)}+ except Exception as e:+ return {"success": False, "error": str(e)}+async def activate_window(self, window_id: int | str) -> Dict[str, Any]:try:if pwc is None:diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.pyindex fc9b6354f8..5a299ac42f 100644--- a/libs/python/computer-server/computer_server/main.py+++ b/libs/python/computer-server/computer_server/main.py@@ -109,6 +109,11 @@ handlers = {"get_application_windows": window_handler.get_application_windows,"get_window_name": window_handler.get_window_name,"get_window_size": window_handler.get_window_size,+ "get_window_position": window_handler.get_window_position,+ "set_window_size": window_handler.set_window_size,+ "set_window_position": window_handler.set_window_position,+ "maximize_window": window_handler.maximize_window,+ "minimize_window": window_handler.minimize_window,"activate_window": window_handler.activate_window,"close_window": window_handler.close_window,# Mouse commandsdiff --git a/libs/python/computer/computer/interface/generic.py b/libs/python/computer/computer/interface/generic.pyindex 80b2419942..e58719dde2 100644--- a/libs/python/computer/computer/interface/generic.py+++ b/libs/python/computer/computer/interface/generic.py@@ -538,6 +538,36 @@ class GenericComputerInterface(BaseComputerInterface):raise RuntimeError(result.get("error", "Failed to get window size"))return int(result.get("width", 0)), int(result.get("height", 0))+ async def get_window_position(self, window_id: int | str) -> tuple[int, int]:+ result = await self._send_command("get_window_position", {"window_id": window_id})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to get window position"))+ return int(result.get("x", 0)), int(result.get("y", 0))++ async def set_window_size(self, window_id: int | str, width: int, height: int) -> None:+ result = await self._send_command(+ "set_window_size", {"window_id": window_id, "width": width, "height": height}+ )+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to set window size"))++ async def set_window_position(self, window_id: int | str, x: int, y: int) -> None:+ result = await self._send_command(+ "set_window_position", {"window_id": window_id, "x": x, "y": y}+ )+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to set window position"))++ async def maximize_window(self, window_id: int | str) -> None:+ result = await self._send_command("maximize_window", {"window_id": window_id})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to maximize window"))++ async def minimize_window(self, window_id: int | str) -> None:+ result = await self._send_command("minimize_window", {"window_id": window_id})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to minimize window"))+async def activate_window(self, window_id: int | str) -> None:result = await self._send_command("activate_window", {"window_id": window_id})if not result.get("success", False):diff --git a/libs/typescript/computer/src/interface/base.ts b/libs/typescript/computer/src/interface/base.tsindex 333060faf1..fe40442100 100644--- a/libs/typescript/computer/src/interface/base.ts+++ b/libs/typescript/computer/src/interface/base.ts@@ -321,6 +321,11 @@ export abstract class BaseComputerInterface {abstract getApplicationWindows(app: string): Promise<Array<number | string>>;abstract getWindowName(windowId: number | string): Promise<string>;abstract getWindowSize(windowId: number | string): Promise<[number, number]>;+ abstract getWindowPosition(windowId: number | string): Promise<[number, number]>;+ abstract setWindowSize(windowId: number | string, width: number, height: number): Promise<void>;+ abstract setWindowPosition(windowId: number | string, x: number, y: number): Promise<void>;+ abstract maximizeWindow(windowId: number | string): Promise<void>;+ abstract minimizeWindow(windowId: number | string): Promise<void>;abstract activateWindow(windowId: number | string): Promise<void>;abstract closeWindow(windowId: number | string): Promise<void>;diff --git a/libs/typescript/computer/src/interface/macos.ts b/libs/typescript/computer/src/interface/macos.tsindex 8b31768754..a113197fdf 100644--- a/libs/typescript/computer/src/interface/macos.ts+++ b/libs/typescript/computer/src/interface/macos.ts@@ -266,6 +266,55 @@ export class MacOSComputerInterface extends BaseComputerInterface {return [Number(response.width) || 0, Number(response.height) || 0];}+ /** Get window position as [x, y]. */+ async getWindowPosition(windowId: number | string): Promise<[number, number]> {+ const response = await this.sendCommand('get_window_position', { window_id: windowId });+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to get window position');+ }+ return [Number(response.x) || 0, Number(response.y) || 0];+ }++ /** Set window size. */+ async setWindowSize(windowId: number | string, width: number, height: number): Promise<void> {+ const response = await this.sendCommand('set_window_size', {+ window_id: windowId,+ width,+ height,+ });+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to set window size');+ }+ }++ /** Set window position. */+ async setWindowPosition(windowId: number | string, x: number, y: number): Promise<void> {+ const response = await this.sendCommand('set_window_position', {+ window_id: windowId,+ x,+ y,+ });+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to set window position');+ }+ }++ /** Maximize a window. */+ async maximizeWindow(windowId: number | string): Promise<void> {+ const response = await this.sendCommand('maximize_window', { window_id: windowId });+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to maximize window');+ }+ }++ /** Minimize a window. */+ async minimizeWindow(windowId: number | string): Promise<void> {+ const response = await this.sendCommand('minimize_window', { window_id: windowId });+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to minimize window');+ }+ }+/** Activate a window by id. */async activateWindow(windowId: number | string): Promise<void> {const response = await this.sendCommand('activate_window', { window_id: windowId });