Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Pinned transformers version, first working grounding version
Dillon DuPontb20d2a0a9384
diff --git a/libs/python/agent/agent/adapters/models/__init__.py b/libs/python/agent/agent/adapters/models/__init__.pyindex 6811c142c7..8a5fb00b27 100644--- a/libs/python/agent/agent/adapters/models/__init__.py+++ b/libs/python/agent/agent/adapters/models/__init__.py@@ -22,7 +22,7 @@ def load_model(model_name: str, device: str = "auto"):)cfg = AutoConfig.from_pretrained(model_name, trust_remote_code=True)cls = cfg.__class__.__name__- print(f"cls: {cls}")+ # print(f"cls: {cls}")if "OpenCUA" in cls:return OpenCUAModel(model_name=model_name, device=device)return GenericHFModel(model_name=model_name, device=device)diff --git a/libs/python/agent/agent/adapters/models/opencua.py b/libs/python/agent/agent/adapters/models/opencua.pyindex f24dfa6b5f..f8abf4a606 100644--- a/libs/python/agent/agent/adapters/models/opencua.py+++ b/libs/python/agent/agent/adapters/models/opencua.py@@ -37,6 +37,7 @@ class OpenCUAModel:torch_dtype="auto",device_map=self.device,trust_remote_code=True,+ attn_implementation="sdpa",)self.image_processor = AutoImageProcessor.from_pretrained(self.model_name, trust_remote_code=Truediff --git a/libs/python/agent/agent/loops/opencua.py b/libs/python/agent/agent/loops/opencua.pyindex c13875b223..a494377b65 100644--- a/libs/python/agent/agent/loops/opencua.py+++ b/libs/python/agent/agent/loops/opencua.py@@ -97,7 +97,7 @@ class OpenCUAConfig(AsyncAgentConfig):},{"type": "text",- "text": instruction+ "text": f"Click on {instruction}"}]}@@ -116,8 +116,7 @@ class OpenCUAConfig(AsyncAgentConfig):# Extract response textoutput_text = response.choices[0].message.content-- print(output_text)+ # print(output_text)# Extract coordinates from pyautogui formatcoordinates = extract_coordinates_from_pyautogui(output_text)diff --git a/libs/python/agent/pyproject.toml b/libs/python/agent/pyproject.tomlindex 811c3a9ca6..0d382fdf20 100644--- a/libs/python/agent/pyproject.toml+++ b/libs/python/agent/pyproject.toml@@ -50,7 +50,7 @@ glm45v-hf = [opencua-hf = ["accelerate","torch",- "transformers>=4.54.0",+ "transformers==4.53.0","tiktoken>=0.11.0","blobfile>=3.0.0"]@@ -75,7 +75,7 @@ all = ["transformers>=4.54.0",# opencua requirements"tiktoken>=0.11.0",- "blobfile>=3.0.0"+ "blobfile>=3.0.0",# ui requirements"gradio>=5.23.3","python-dotenv>=1.0.1",