Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fixed missing screenshots in agent
Dillon DuPontfa888361321e
diff --git a/libs/python/agent/agent/integrations/hud/agent.py b/libs/python/agent/agent/integrations/hud/agent.pyindex 9156cf4a51..8ade909c73 100644--- a/libs/python/agent/agent/integrations/hud/agent.py+++ b/libs/python/agent/agent/integrations/hud/agent.py@@ -142,7 +142,20 @@ class ComputerAgent(Agent[BaseComputerAgent, dict[str, Any]]):else:message = f"{self.base_system_prompt}\n\nPlease analyze the current screen and determine what action to take."- self.conversation_history.append({"role": "user", "content": message})+ input_content = [+ {"type": "input_text", "text": message}+ ]++ # Add screenshot if present+ if observation.screenshot:+ input_content.append(+ {+ "type": "input_image",+ "image_url": f"data:image/png;base64,{observation.screenshot}",+ }+ )++ self.conversation_history.append({"role": "user", "content": input_content})else:# Subsequent interactions - check if last action was computer_call# If so, add computer_call_output with screenshot instead of user message@@ -176,7 +189,20 @@ class ComputerAgent(Agent[BaseComputerAgent, dict[str, Any]]):else:# No computer_call found, add regular user messagemessage = "Continue with the task based on the current screen state."- self.conversation_history.append({"role": "user", "content": message})+ input_content = [+ {"type": "input_text", "text": message}+ ]++ # Add screenshot if present+ if observation.screenshot:+ input_content.append(+ {+ "type": "input_image",+ "image_url": f"data:image/png;base64,{observation.screenshot}",+ }+ )++ self.conversation_history.append({"role": "user", "content": input_content})# Run ComputerAgenttry: