Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Fix Pylance lint
f-trycuacc3891a7ad14
diff --git a/examples/agent_examples.py b/examples/agent_examples.pyindex d17045b734..2cf1158790 100644--- a/examples/agent_examples.py+++ b/examples/agent_examples.py@@ -32,10 +32,10 @@ async def run_omni_agent_example():# Create agent with loop and provideragent = ComputerAgent(computer=computer,- # loop=AgentLoop.ANTHROPIC,- loop=AgentLoop.OMNI,- model=LLM(provider=LLMProvider.OPENAI, name="gpt-4.5-preview"),- # model=LLM(provider=LLMProvider.ANTHROPIC, name="claude-3-7-sonnet-20250219"),+ loop=AgentLoop.ANTHROPIC,+ # loop=AgentLoop.OMNI,+ # model=LLM(provider=LLMProvider.OPENAI, name="gpt-4.5-preview"),+ model=LLM(provider=LLMProvider.ANTHROPIC, name="claude-3-7-sonnet-20250219"),save_trajectory=True,trajectory_dir=str(Path("trajectories")),only_n_most_recent_images=3,diff --git a/examples/pylume_examples.py b/examples/pylume_examples.pyindex 6700c99b6b..319396863a 100644--- a/examples/pylume_examples.py+++ b/examples/pylume_examples.py@@ -1,12 +1,6 @@import asyncio-from pylume import (- PyLume,- ImageRef,- VMRunOpts,- SharedDirectory,- VMConfig,- VMUpdateOpts-)+from pylume import PyLume, ImageRef, VMRunOpts, SharedDirectory, VMConfig, VMUpdateOpts+async def main():"""Example usage of PyLume."""@@ -22,11 +16,11 @@ async def main():vm_config = VMConfig(name="lume-vm-new",os="macOS",- cpu=2,+ cpu=2,memory="4GB",- disk_size="64GB",+ disk_size="64GB", # type: ignoredisplay="1024x768",- ipsw="latest"+ ipsw="latest",)await pylume.create_vm(vm_config)@@ -39,7 +33,7 @@ async def main():print("\n=== Listing Available Images ===")images = await pylume.get_images()print("Available Images:", images)-+# List all VMs to verify creationprint("\n=== Listing All VMs ===")vms = await pylume.list_vms()@@ -52,30 +46,21 @@ async def main():# Update VM settingsprint("\n=== Updating VM Settings ===")- update_opts = VMUpdateOpts(- cpu=8,- memory="4GB"- )+ update_opts = VMUpdateOpts(cpu=8, memory="4GB")await pylume.update_vm("lume-vm", update_opts)- # Pull an image+ # Pull an imageimage_ref = ImageRef(- image="macos-sequoia-vanilla",- tag="latest",- registry="ghcr.io",- organization="trycua"+ image="macos-sequoia-vanilla", tag="latest", registry="ghcr.io", organization="trycua")await pylume.pull_image(image_ref, name="lume-vm-pulled")# Run with shared directoryrun_opts = VMRunOpts(- no_display=False,- shared_directories=[- SharedDirectory(- host_path="~/shared",- read_only=False- )- ]+ no_display=False, # type: ignore+ shared_directories=[ # type: ignore+ SharedDirectory(host_path="~/shared", read_only=False) # type: ignore+ ],)await pylume.run_vm("lume-vm", run_opts)@@ -94,5 +79,6 @@ async def main():print("\n=== Deleting VM ===")await pylume.delete_vm("lume-vm-cloned")-if __name__ == '__main__':++if __name__ == "__main__":asyncio.run(main())diff --git a/libs/agent/agent/providers/anthropic/tools/bash.py b/libs/agent/agent/providers/anthropic/tools/bash.pyindex 00bdd572a6..703327e697 100644--- a/libs/agent/agent/providers/anthropic/tools/bash.py+++ b/libs/agent/agent/providers/anthropic/tools/bash.py@@ -74,31 +74,30 @@ class _BashSession:async with asyncio.timeout(self._timeout):while True:await asyncio.sleep(self._output_delay)- # if we read directly from stdout/stderr, it will wait forever for- # EOF. use the StreamReader buffer directly instead.- output = (- self._process.stdout._buffer.decode()- ) # pyright: ignore[reportAttributeAccessIssue]- if self._sentinel in output:- # strip the sentinel and break- output = output[: output.index(self._sentinel)]- break+ # Read from stdout using the proper API+ output_bytes = await self._process.stdout.read()+ if output_bytes:+ output = output_bytes.decode()+ if self._sentinel in output:+ # strip the sentinel and break+ output = output[: output.index(self._sentinel)]+ breakexcept asyncio.TimeoutError:self._timed_out = Trueraise ToolError(f"timed out: bash has not returned in {self._timeout} seconds and must be restarted",) from None- if output.endswith("\n"):+ if output and output.endswith("\n"):output = output[:-1]- error = self._process.stderr._buffer.decode() # pyright: ignore[reportAttributeAccessIssue]- if error.endswith("\n"):+ # Read from stderr using the proper API+ error_bytes = await self._process.stderr.read()+ error = error_bytes.decode() if error_bytes else ""+ if error and error.endswith("\n"):error = error[:-1]- # clear the buffers so that the next output can be read correctly- self._process.stdout._buffer.clear() # pyright: ignore[reportAttributeAccessIssue]- self._process.stderr._buffer.clear() # pyright: ignore[reportAttributeAccessIssue]+ # No need to clear buffers as we're using read() which consumes the datareturn CLIResult(output=output, error=error)diff --git a/libs/agent/agent/providers/omni/types.py b/libs/agent/agent/providers/omni/types.pyindex 3c442ea55d..734a4af603 100644--- a/libs/agent/agent/providers/omni/types.py+++ b/libs/agent/agent/providers/omni/types.py@@ -11,10 +11,6 @@ class LLMProvider(StrEnum):ANTHROPIC = "anthropic"OPENAI = "openai"--LLMProvider--@dataclassclass LLM:"""Configuration for LLM model and provider."""diff --git a/libs/computer/computer/computer.py b/libs/computer/computer/computer.pyindex 5aa851e6f1..26fcbf9b4c 100644--- a/libs/computer/computer/computer.py+++ b/libs/computer/computer/computer.py@@ -5,9 +5,7 @@ from pylume.models import (VMUpdateOpts,ImageRef,SharedDirectory,- VMStatus,- VMConfig,- CloneSpec,+ VMStatus)import asynciofrom .models import Computer as ComputerConfig, Display@@ -15,7 +13,6 @@ from .interface.factory import InterfaceFactoryimport timefrom PIL import Imageimport io-from .utils import bytes_to_imageimport refrom .logger import Logger, LogLevelimport json@@ -266,7 +263,8 @@ class Computer:# Log the equivalent curl command for debuggingpayload = json.dumps({"noDisplay": False, "sharedDirectories": []})curl_cmd = f"curl -X POST 'http://localhost:3000/lume/vms/{self.config.name}/run' -H 'Content-Type: application/json' -d '{payload}'"- print(f"\nEquivalent curl command:\n{curl_cmd}\n")+ self.logger.info(f"Equivalent curl command:")+ self.logger.info(f"{curl_cmd}")try:response = await self.config.pylume.run_vm(self.config.name, run_opts) # type: ignore[attr-defined]diff --git a/libs/som/som/detect.py b/libs/som/som/detect.pyindex e845740e7f..10a21804e3 100644--- a/libs/som/som/detect.py+++ b/libs/som/som/detect.py@@ -51,6 +51,7 @@ def process_text_box(box, image):"""Process a single text box with OCR."""try:import easyocr+ from typing import List, Tuple, Any, Sequencex1 = int(min(point[0] for point in box))y1 = int(min(point[1] for point in box))@@ -67,12 +68,15 @@ def process_text_box(box, image):region = image[y1:y2, x1:x2]if region.size > 0:reader = easyocr.Reader(["en"])- result = reader.readtext(region)- if result:- text = result[0][1] # Get text- conf = result[0][2] # Get confidence- if conf > 0.5:- return text, [x1, y1, x2, y2], conf+ results = reader.readtext(region)+ if results and len(results) > 0:+ # EasyOCR returns a list of tuples (bbox, text, confidence)+ first_result = results[0]+ if isinstance(first_result, (list, tuple)) and len(first_result) >= 3:+ text = str(first_result[1])+ confidence = float(first_result[2])+ if confidence > 0.5:+ return text, [x1, y1, x2, y2], confidenceexcept Exception:passreturn None@@ -90,6 +94,10 @@ def check_ocr_box(image_path: Union[str, Path]) -> Tuple[List[str], List[List[fllogger.error(f"Failed to read image: {image_path}")return [], []+ # Get image dimensions+ img_height, img_width = image_cv.shape[:2]+ confidence_threshold = 0.5+# Use EasyOCRimport sslimport easyocr@@ -120,7 +128,7 @@ def check_ocr_box(image_path: Union[str, Path]) -> Tuple[List[str], List[List[flx2 = max(point[0] for point in box)y2 = max(point[1] for point in box)- if conf > 0.5: # Only keep higher confidence detections+ if float(conf) > 0.5: # Only keep higher confidence detectionstexts.append(text)boxes.append([x1, y1, x2, y2])diff --git a/libs/som/som/ocr.py b/libs/som/som/ocr.pyindex 245fc9dce4..a206e057b8 100644--- a/libs/som/som/ocr.py+++ b/libs/som/som/ocr.py@@ -1,4 +1,4 @@-from typing import List, Dict, Any, Tuple+from typing import List, Dict, Any, Tuple, Unionimport loggingimport signalfrom contextlib import contextmanager@@ -137,14 +137,17 @@ class OCRProcessor:img_width, img_height = image.sizefor box, text, conf in results:- if conf < confidence_threshold:+ # Ensure conf is float+ conf_float = float(conf)+ if conf_float < confidence_threshold:continue# Convert box format to [x1, y1, x2, y2]- x1 = min(point[0] for point in box) / img_width- y1 = min(point[1] for point in box) / img_height- x2 = max(point[0] for point in box) / img_width- y2 = max(point[1] for point in box) / img_height+ # Ensure box points are properly typed as float+ x1 = min(float(point[0]) for point in box) / img_width+ y1 = min(float(point[1]) for point in box) / img_height+ x2 = max(float(point[0]) for point in box) / img_width+ y2 = max(float(point[1]) for point in box) / img_heightdetections.append({diff --git a/libs/som/som/util/utils.py b/libs/som/som/util/utils.pyindex 6658b69a45..4303b91d6b 100644--- a/libs/som/som/util/utils.py+++ b/libs/som/som/util/utils.py@@ -3,7 +3,7 @@ import cv2import matplotlib.pyplot as pltimport numpy as npfrom PIL import Image-from typing import Union+from typing import Union, List, Tuple, Any, Optional, cast, Sequenceimport timeimport signalfrom contextlib import contextmanager@@ -56,7 +56,7 @@ def check_ocr_box(goal_filtering=None,easyocr_args=None,use_paddleocr=False,-):+) -> Tuple[Tuple[List[str], List[Tuple[float, float, float, float]]], Optional[Any]]:"""Check OCR box using EasyOCR with optimized settings.Args:@@ -66,6 +66,11 @@ def check_ocr_box(goal_filtering: Optional filtering of resultseasyocr_args: Arguments for EasyOCRuse_paddleocr: Ignored (kept for backward compatibility)++ Returns:+ Tuple containing:+ - Tuple of (text_list, bounding_boxes)+ - goal_filtering value"""logger.info("Starting OCR processing...")start_time = time.time()@@ -104,9 +109,11 @@ def check_ocr_box(# Use EasyOCR with timeoutlogger.info("Starting EasyOCR detection with 5 second timeout...")with timeout(5): # 5 second timeout- result = reader.readtext(image_np, **default_args)- coord = [item[0] for item in result]- text = [item[1] for item in result]+ # EasyOCR's readtext returns a list of tuples, where each tuple is (bbox, text, confidence)+ raw_result = reader.readtext(image_np, **default_args)+ result = cast(Sequence[Tuple[List[Tuple[float, float]], str, float]], raw_result)+ coord = [item[0] for item in result] # item[0] is the bbox coordinates+ text = [item[1] for item in result] # item[1] is the text contentlogger.info(f"OCR completed successfully. Found {len(text)} text regions")logger.info(f"Detected text: {text}")@@ -129,7 +136,16 @@ def check_ocr_box(for item in coord:x, y, a, b = get_xywh(item)bb.append((x, y, a, b))- cv2.rectangle(opencv_img, (x, y), (x + a, y + b), (0, 255, 0), 2)+ # Convert float coordinates to integers for cv2.rectangle+ x_val = cast(float, x)+ y_val = cast(float, y)+ a_val = cast(float, a)+ b_val = cast(float, b)+ x_int, y_int = int(x_val), int(y_val)+ a_int, b_int = int(a_val), int(b_val)+ cv2.rectangle(+ opencv_img, (x_int, y_int), (x_int + a_int, y_int + b_int), (0, 255, 0), 2+ )plt.imshow(cv2.cvtColor(opencv_img, cv2.COLOR_BGR2RGB))else:if output_bb_format == "xywh":@@ -137,6 +153,9 @@ def check_ocr_box(elif output_bb_format == "xyxy":bb = [get_xyxy(item) for item in coord]+ # Cast the bounding boxes to the expected type+ bb = cast(List[Tuple[float, float, float, float]], bb)+logger.info("OCR processing complete")return (text, bb), goal_filtering