Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Add get_desktop_environment, set_wallpaper
Dillon DuPontfb174d6aa4e5
diff --git a/docs/content/docs/computer-sdk/commands.mdx b/docs/content/docs/computer-sdk/commands.mdxindex d8e804935c..4d36baa4c0 100644--- a/docs/content/docs/computer-sdk/commands.mdx+++ b/docs/content/docs/computer-sdk/commands.mdx@@ -13,16 +13,12 @@ Execute shell commands and get detailed results:<Tabs items={['Python', 'TypeScript']}><Tab value="Python">- ```python- # Run shell command result = await- computer.interface.run_command(cmd) # result.stdout, result.stderr, result.returncode- ```+ ```python # Run shell command result = await computer.interface.run_command(cmd) #+ result.stdout, result.stderr, result.returncode ```</Tab><Tab value="TypeScript">- ```typescript- // Run shell command const result = await- computer.interface.runCommand(cmd); // result.stdout, result.stderr, result.returncode- ```+ ```typescript // Run shell command const result = await computer.interface.runCommand(cmd); //+ result.stdout, result.stderr, result.returncode ```</Tab></Tabs>@@ -108,20 +104,14 @@ Mouse wheel and scrolling control:<Tabs items={['Python', 'TypeScript']}><Tab value="Python">- ```python- # Scrolling- await computer.interface.scroll(x, y) # Scroll the mouse wheel- await computer.interface.scroll_down(clicks) # Scroll down await- computer.interface.scroll_up(clicks) # Scroll up- ```+ ```python # Scrolling await computer.interface.scroll(x, y) # Scroll the mouse wheel await+ computer.interface.scroll_down(clicks) # Scroll down await computer.interface.scroll_up(clicks)+ # Scroll up ```</Tab><Tab value="TypeScript">- ```typescript- // Scrolling- await computer.interface.scroll(x, y); // Scroll the mouse wheel- await computer.interface.scrollDown(clicks); // Scroll down- await computer.interface.scrollUp(clicks); // Scroll up- ```+ ```typescript // Scrolling await computer.interface.scroll(x, y); // Scroll the mouse wheel+ await computer.interface.scrollDown(clicks); // Scroll down await+ computer.interface.scrollUp(clicks); // Scroll up ```</Tab></Tabs>@@ -149,6 +139,35 @@ Screen capture and display information:</Tab></Tabs>+## Desktop Actions++Control desktop environment features like wallpaper:++<Tabs items={['Python', 'TypeScript']}>+ <Tab value="Python">+ ```python+ # Get current desktop environment (e.g., 'xfce4', 'gnome', 'kde', 'mac', 'windows')+ env = await computer.interface.get_desktop_environment()+ print(env) # "xfce4"++ # Set desktop wallpaper to an image file accessible on the VM+ await computer.interface.set_wallpaper("/home/cua/shared/wallpaper.png")+ ```++ </Tab>+ <Tab value="TypeScript">+ ```typescript+ // Get current desktop environment+ const env = await computer.interface.getDesktopEnvironment();+ print(env) # "xfce4"++ // Set desktop wallpaper to an image file accessible on the VM+ await computer.interface.setWallpaper('/home/cua/shared/wallpaper.png');+ ```++ </Tab>+</Tabs>+## Clipboard ActionsSystem clipboard management:@@ -240,7 +259,7 @@ Access system accessibility information:// Get accessibility treeawait computer.interface.getAccessibilityTree();-```+````</Tab></Tabs>@@ -277,4 +296,5 @@ Manage Python environments:```</Tab>-</Tabs>\ No newline at end of file+</Tabs>+````diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.pyindex 42b27a0cf9..73250d444b 100644--- a/libs/python/computer-server/computer_server/handlers/base.py+++ b/libs/python/computer-server/computer_server/handlers/base.py@@ -85,6 +85,26 @@ class BaseFileHandler(ABC):pass+class BaseDesktopHandler(ABC):+ """Abstract base class for OS-specific desktop handlers.++ Categories:+ - Wallpaper Actions: Methods for wallpaper operations+ - Desktop shortcut actions: Methods for managing desktop shortcuts+ """++ # Wallpaper Actions+ @abstractmethod+ async def get_desktop_environment(self) -> Dict[str, Any]:+ """Get the current desktop environment name."""+ pass++ @abstractmethod+ async def set_wallpaper(self, path: str) -> Dict[str, Any]:+ """Set the desktop wallpaper to the file at path."""+ pass++class BaseAutomationHandler(ABC):"""Abstract base class for OS-specific automation handlers.diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.pyindex 3c90935f14..77d88e5f17 100644--- a/libs/python/computer-server/computer_server/handlers/factory.py+++ b/libs/python/computer-server/computer_server/handlers/factory.py@@ -4,7 +4,12 @@ from typing import Tuple, Typefrom computer_server.diorama.base import BaseDioramaHandler-from .base import BaseAccessibilityHandler, BaseAutomationHandler, BaseFileHandler+from .base import (+ BaseAccessibilityHandler,+ BaseAutomationHandler,+ BaseDesktopHandler,+ BaseFileHandler,+)# Conditionally import platform-specific handlerssystem = platform.system().lower()@@ -17,7 +22,7 @@ elif system == "linux":elif system == "windows":from .windows import WindowsAccessibilityHandler, WindowsAutomationHandler-from .generic import GenericFileHandler+from .generic import GenericDesktopHandler, GenericFileHandlerclass HandlerFactory:@@ -50,7 +55,13 @@ class HandlerFactory:@staticmethoddef create_handlers() -> (- Tuple[BaseAccessibilityHandler, BaseAutomationHandler, BaseDioramaHandler, BaseFileHandler]+ Tuple[+ BaseAccessibilityHandler,+ BaseAutomationHandler,+ BaseDioramaHandler,+ BaseFileHandler,+ BaseDesktopHandler,+ ]):"""Create and return appropriate handlers for the current OS.@@ -70,6 +81,7 @@ class HandlerFactory:MacOSAutomationHandler(),MacOSDioramaHandler(),GenericFileHandler(),+ GenericDesktopHandler(),)elif os_type == "linux":return (@@ -77,6 +89,7 @@ class HandlerFactory:LinuxAutomationHandler(),BaseDioramaHandler(),GenericFileHandler(),+ GenericDesktopHandler(),)elif os_type == "windows":return (@@ -84,6 +97,7 @@ class HandlerFactory:WindowsAutomationHandler(),BaseDioramaHandler(),GenericFileHandler(),+ GenericDesktopHandler(),)else:raise NotImplementedError(f"OS '{os_type}' is not supported")diff --git a/libs/python/computer-server/computer_server/handlers/generic.py b/libs/python/computer-server/computer_server/handlers/generic.pyindex da29f1c384..c73483129e 100644--- a/libs/python/computer-server/computer_server/handlers/generic.py+++ b/libs/python/computer-server/computer_server/handlers/generic.py@@ -2,6 +2,7 @@Generic handlers for all OSes.Includes:+- DesktopHandler- FileHandler"""@@ -10,7 +11,8 @@ import base64from pathlib import Pathfrom typing import Any, Dict, Optional-from .base import BaseFileHandler+from ..utils import wallpaper+from .base import BaseDesktopHandler, BaseFileHandlerdef resolve_path(path: str) -> Path:@@ -25,6 +27,46 @@ def resolve_path(path: str) -> Path:return Path(path).expanduser().resolve()+class GenericDesktopHandler(BaseDesktopHandler):+ """+ Generic desktop handler providing desktop-related operations.++ Implements:+ - get_desktop_environment: detect current desktop environment+ - set_wallpaper: set desktop wallpaper path+ """++ async def get_desktop_environment(self) -> Dict[str, Any]:+ """+ Get the current desktop environment.++ Returns:+ Dict containing 'success' boolean and either 'environment' string or 'error' string+ """+ try:+ env = wallpaper.get_desktop_environment()+ return {"success": True, "environment": env}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def set_wallpaper(self, path: str) -> Dict[str, Any]:+ """+ Set the desktop wallpaper to the specified path.++ Args:+ path: The file path to set as wallpaper++ Returns:+ Dict containing 'success' boolean and optionally 'error' string+ """+ try:+ file_path = resolve_path(path)+ ok = wallpaper.set_wallpaper(str(file_path))+ return {"success": bool(ok)}+ except Exception as e:+ return {"success": False, "error": str(e)}++class GenericFileHandler(BaseFileHandler):"""Generic file handler that provides file system operations for all operating systems.diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.pyindex 9c0c052bfa..005c7ddb64 100644--- a/libs/python/computer-server/computer_server/main.py+++ b/libs/python/computer-server/computer_server/main.py@@ -75,7 +75,7 @@ except Exception:except Exception:package_version = "unknown"-accessibility_handler, automation_handler, diorama_handler, file_handler = (+accessibility_handler, automation_handler, diorama_handler, file_handler, desktop_handler = (HandlerFactory.create_handlers())handlers = {@@ -99,6 +99,9 @@ handlers = {"delete_file": file_handler.delete_file,"create_dir": file_handler.create_dir,"delete_dir": file_handler.delete_dir,+ # Desktop commands+ "get_desktop_environment": desktop_handler.get_desktop_environment,+ "set_wallpaper": desktop_handler.set_wallpaper,# Mouse commands"mouse_down": automation_handler.mouse_down,"mouse_up": automation_handler.mouse_up,diff --git a/libs/python/computer/computer/interface/base.py b/libs/python/computer/computer/interface/base.pyindex 8fa4002519..42c7ab0f82 100644--- a/libs/python/computer/computer/interface/base.py+++ b/libs/python/computer/computer/interface/base.py@@ -436,6 +436,26 @@ class BaseComputerInterface(ABC):"""pass+ # Desktop actions+ @abstractmethod+ async def get_desktop_environment(self) -> str:+ """Get the current desktop environment.++ Returns:+ The name of the current desktop environment.+ """+ pass++ @abstractmethod+ async def set_wallpaper(self, path: str) -> None:+ """Set the desktop wallpaper to the specified path.++ Args:+ path: The file path to set as wallpaper+ """+ pass++ # Shell actions@abstractmethodasync def run_command(self, command: str) -> CommandResult:"""Run shell command and return structured result.diff --git a/libs/python/computer/computer/interface/generic.py b/libs/python/computer/computer/interface/generic.pyindex 7cd2372f08..7cf47461e5 100644--- a/libs/python/computer/computer/interface/generic.py+++ b/libs/python/computer/computer/interface/generic.py@@ -487,6 +487,18 @@ class GenericComputerInterface(BaseComputerInterface):raise RuntimeError(result.get("error", "Failed to list directory"))return result.get("files", [])+ # Desktop actions+ async def get_desktop_environment(self) -> str:+ result = await self._send_command("get_desktop_environment")+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to get desktop environment"))+ return result.get("environment", "unknown")++ async def set_wallpaper(self, path: str) -> None:+ result = await self._send_command("set_wallpaper", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to set wallpaper"))+# Command executionasync def run_command(self, command: str) -> CommandResult:result = await self._send_command("run_command", {"command": command})diff --git a/libs/typescript/computer/src/interface/base.ts b/libs/typescript/computer/src/interface/base.tsindex 4cd85f1879..e41c4416ec 100644--- a/libs/typescript/computer/src/interface/base.ts+++ b/libs/typescript/computer/src/interface/base.ts@@ -314,6 +314,10 @@ export abstract class BaseComputerInterface {abstract getScreenSize(): Promise<ScreenSize>;abstract getCursorPosition(): Promise<CursorPosition>;+ // Desktop Actions+ abstract getDesktopEnvironment(): Promise<string>;+ abstract setWallpaper(path: string): Promise<void>;+// Clipboard Actionsabstract copyToClipboard(): Promise<string>;abstract setClipboard(text: string): Promise<void>;diff --git a/libs/typescript/computer/src/interface/macos.ts b/libs/typescript/computer/src/interface/macos.tsindex c1c05454a1..86522042e2 100644--- a/libs/typescript/computer/src/interface/macos.ts+++ b/libs/typescript/computer/src/interface/macos.ts@@ -3,8 +3,8 @@*/import type { ScreenSize } from '../types';-import { BaseComputerInterface } from './base';import type { AccessibilityNode, CursorPosition, MouseButton } from './base';+import { BaseComputerInterface } from './base';export class MacOSComputerInterface extends BaseComputerInterface {// Mouse Actions@@ -212,6 +212,29 @@ export class MacOSComputerInterface extends BaseComputerInterface {return response.position as CursorPosition;}+ // Desktop Actions+ /**+ * Get the current desktop environment string (e.g., 'xfce4', 'gnome', 'kde', 'mac', 'windows').+ */+ async getDesktopEnvironment(): Promise<string> {+ const response = await this.sendCommand('get_desktop_environment');+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to get desktop environment');+ }+ return (response.environment as string) || 'unknown';+ }++ /**+ * Set the desktop wallpaper image.+ * @param path Absolute path to the image file on the VM+ */+ async setWallpaper(path: string): Promise<void> {+ const response = await this.sendCommand('set_wallpaper', { path });+ if (!response.success) {+ throw new Error((response.error as string) || 'Failed to set wallpaper');+ }+ }+// Clipboard Actions/*** Copy current selection to clipboard and return the content.