Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added standardized logging for easy parsing of trajectories
Dillon DuPontcf7d05421239
diff --git a/examples/video_maker_traj.py b/examples/video_maker_traj.pyindex 34a5ad3c2b..b9966aa0ff 100644--- a/examples/video_maker_traj.py+++ b/examples/video_maker_traj.py@@ -68,82 +68,148 @@ def load_cursor_images():last_known_cursor_position = Nonelast_known_thought = None-def extract_thought_from_api_response(filename):- """Extract thought from API response for the current frame."""- global last_known_thought+def parse_agent_response(filename_or_turn_dir):+ """Parse agent response JSON file to extract text, actions, and cursor positions."""++ # Check if we're getting a filename or turn directory+ if os.path.isdir(filename_or_turn_dir):+ turn_dir = filename_or_turn_dir+ else:+ turn_dir = os.path.dirname(filename_or_turn_dir)++ # Find agent response files in the turn directory+ agent_response_files = [f for f in os.listdir(turn_dir) if f.endswith('_agent_response.json')]- turn_dir = os.path.dirname(filename)- api_response_files = [f for f in os.listdir(turn_dir) if f.endswith('_response.json')]+ result = {+ "text": [],+ "actions": [],+ "cursor_positions": []+ }- for api_file in api_response_files:+ for agent_file in agent_response_files:try:- with open(os.path.join(turn_dir, api_file), 'r') as f:+ with open(os.path.join(turn_dir, agent_file), 'r') as f:data = json.load(f)- # Extract content from response- content = data.get('response', {}).get('choices', [{}])[0].get('message', {}).get('content', '')+ response_data = data.get('response', {})++ # First check for content field (simple text response)+ if response_data.get("content"):+ result["text"].append(response_data.get("content", ""))- # Extract the Thought section- thought_match = re.search(r"Thought: (.*?)(?:\nAction:|$)", content, re.DOTALL)- if thought_match:- thought = thought_match.group(1).strip()- if thought:- last_known_thought = thought- return thought- except (json.JSONDecodeError, FileNotFoundError, KeyError):- pass+ # Process outputs array if present+ outputs = response_data.get("output", [])+ for output in outputs:+ output_type = output.get("type")++ if output_type == "message":+ content = output.get("content", [])+ for content_part in content:+ if content_part.get("text"):+ result["text"].append(content_part.get("text", ""))++ elif output_type == "reasoning":+ # Handle reasoning (thought) content+ summary_content = output.get("summary", [])+ if summary_content:+ for summary_part in summary_content:+ if summary_part.get("type") == "summary_text":+ result["text"].append(summary_part.get("text", ""))+ else:+ summary_text = output.get("text", "")+ if summary_text:+ result["text"].append(summary_text)++ elif output_type == "computer_call":+ action = output.get("action", {})+ if action:+ result["actions"].append(action)+ # Extract cursor position if available+ if action.get("x") is not None and action.get("y") is not None:+ result["cursor_positions"].append((action.get("x"), action.get("y")))+ except Exception as e:+ print(f"Error processing {agent_file}: {e}")++ return result++def extract_thought_from_agent_response(filename_or_turn_dir):+ """Extract thought from agent response for the current frame."""+ global last_known_thought++ agent_response = parse_agent_response(filename_or_turn_dir)++ if agent_response["text"]:+ # Use the first text entry as the thought+ last_known_thought = agent_response["text"][0]+ return last_known_thought# Return the last known thought if no new thought is foundreturn last_known_thought-def extract_cursor_position_from_filename(filename):- """Extract cursor position from a filename containing click info."""+def extract_cursor_position_from_agent_response(filename_or_turn_dir):+ """Extract cursor position from agent response."""global last_known_cursor_position- # For 'screenshot_NNN_click_TIMESTAMP.png', try to extract coordinates- match = re.search(r'click_(\d+)_(\d+)_\d+\.png$', filename)- if match:- position = (int(match.group(1)), int(match.group(2)))- last_known_cursor_position = position- return position+ # Check if we're getting a filename or turn directory+ if os.path.isdir(filename_or_turn_dir):+ turn_dir = filename_or_turn_dir+ else:+ turn_dir = os.path.dirname(filename_or_turn_dir)- # Check if we have position info from API response- turn_dir = os.path.dirname(filename)- api_response_files = [f for f in os.listdir(turn_dir) if f.endswith('_response.json')]+ # Find agent response files in the turn directory+ agent_response_files = [f for f in os.listdir(turn_dir) if f.endswith('_agent_response.json')]- for api_file in api_response_files:+ for agent_file in agent_response_files:try:- with open(os.path.join(turn_dir, api_file), 'r') as f:+ with open(os.path.join(turn_dir, agent_file), 'r') as f:data = json.load(f)- # Extract action from response- content = data.get('response', {}).get('choices', [{}])[0].get('message', {}).get('content', '')- # Look for coordinates in the action- # First try the pattern from the example: click(start_box='(28,15)')- coord_match = re.search(r"click\(start_box='\((\d+),(\d+)\)'\)", content)- if coord_match:- position = (int(coord_match.group(1)), int(coord_match.group(2)))- last_known_cursor_position = position- return position+ response_data = data.get('response', {})- # Try alternative pattern: click(start_box='<|box_start|>(x,y)<|box_end|>')- alt_match = re.search(r"click\(start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'\)", content)- if alt_match:- position = (int(alt_match.group(1)), int(alt_match.group(2)))- last_known_cursor_position = position- return position- except (json.JSONDecodeError, FileNotFoundError, KeyError):- pass-- # No new position found, return the last known position+ # Process outputs array if present+ outputs = response_data.get("output", [])+ for output in outputs:+ if output.get("type") == "computer_call":+ action = output.get("action", {})+ if action.get("x") is not None and action.get("y") is not None:+ position = (action.get("x"), action.get("y"))+ last_known_cursor_position = position+ return position+ except Exception as e:+ print(f"Error processing {agent_file}: {e}")++ # No position found in agent response, return the last known positionreturn last_known_cursor_position-def extract_action_from_filename(filename):- """Determine the action type from the filename pattern."""- if 'click' in filename:- return "clicking"- elif 'type' in filename:- return "typing"+def extract_action_from_agent_response(filename_or_turn_dir):+ """Determine the action type from agent response."""+ # Check if we're getting a filename or turn directory+ if os.path.isdir(filename_or_turn_dir):+ turn_dir = filename_or_turn_direlse:- return "normal"+ turn_dir = os.path.dirname(filename_or_turn_dir)++ # Find agent response files in the turn directory+ agent_response_files = [f for f in os.listdir(turn_dir) if f.endswith('_agent_response.json')]++ for agent_file in agent_response_files:+ try:+ with open(os.path.join(turn_dir, agent_file), 'r') as f:+ data = json.load(f)+ response_data = data.get('response', {})++ # Process outputs array if present+ outputs = response_data.get("output", [])+ for output in outputs:+ if output.get("type") == "computer_call":+ action = output.get("action", {})+ action_type = action.get("type", "")+ if action_type == "click":+ return "clicking"+ elif action_type == "type" or action_type == "input":+ return "typing"+ except Exception as e:+ print(f"Error processing {agent_file}: {e}")++ return "normal"def create_animated_vignette(image, frame_index):"""@@ -451,58 +517,54 @@ def create_cursor_overlay(base_image, position, cursor_images, thought_text=Nonereturn result-def get_screenshot_files(trajectory_dir):+def get_turns(trajectory_dir):"""- Get all screenshot files from a trajectory directory, sorted by sequence number.+ Get all turn folders from a trajectory directory and their corresponding files.Args:- trajectory_dir: Path to trajectory directory containing turn_XXX folders+ trajectory_dir: Path to trajectory directoryReturns:- List of tuples (path, sequence_number, action_type, position)+ List of tuples (turn_dir, agent_response_path, image_file_path)"""- screenshot_files = []+ turns = []# List all turn directories in orderturn_dirs = sorted([d for d in os.listdir(trajectory_dir) if d.startswith('turn_')],- key=lambda x: int(x.split('_')[1]))+ key=lambda x: int(x.split('_')[1]))- for turn_dir in turn_dirs:- turn_path = os.path.join(trajectory_dir, turn_dir)+ for turn_dir_name in turn_dirs:+ turn_path = os.path.join(trajectory_dir, turn_dir_name)if not os.path.isdir(turn_path):continue-- # Get all screenshot files in this turn- files = [f for f in os.listdir(turn_path) if f.startswith('screenshot_') and f.endswith('.png')]- for file in files:- file_path = os.path.join(turn_path, file)-- # Extract sequence number from filename (e.g., screenshot_003_...)- seq_match = re.search(r'screenshot_(\d+)', file)- if seq_match:- seq_number = int(seq_match.group(1))-- # Determine action type from filename- action_type = extract_action_from_filename(file)-- # Get cursor position if available- position = extract_cursor_position_from_filename(file_path)-- screenshot_files.append((file_path, seq_number, action_type, position))-- # Sort by sequence number- screenshot_files.sort(key=lambda x: x[1])+ # Find agent response files (if any)+ agent_response_files = [f for f in os.listdir(turn_path) if f.endswith('_agent_response.json')]+ agent_response_path = None+ if agent_response_files:+ agent_response_path = os.path.join(turn_path, agent_response_files[0])++ # Find screenshot files (if any)+ screenshot_files = [f for f in os.listdir(turn_path) if f.startswith('screenshot_') and f.endswith('.png')]+ screenshot_path = None+ if screenshot_files:+ # Sort by sequence number to get the main one+ sorted_screenshots = sorted(screenshot_files,+ key=lambda x: int(re.search(r'screenshot_(\d+)', x).group(1)+ if re.search(r'screenshot_(\d+)', x) else 0))+ screenshot_path = os.path.join(turn_path, sorted_screenshots[0]) if sorted_screenshots else None++ turns.append((turn_path, agent_response_path, screenshot_path))- return screenshot_files+ return turnsdef process_trajectory(trajectory_dir, output_dir, cursors):"""Process a trajectory directory and create output frames."""- # Get all screenshot files- screenshot_files = get_screenshot_files(trajectory_dir)+ # Get all turns with their associated files+ turns = get_turns(trajectory_dir)- if not screenshot_files:- print(f"No screenshot files found in {trajectory_dir}")+ if not turns:+ print(f"No turn directories found in {trajectory_dir}")return# Create output directory@@ -511,20 +573,27 @@ def process_trajectory(trajectory_dir, output_dir, cursors):# Track frame indexframe_index = 0- # Process each screenshot+ # Process each turnprev_img = Noneprev_cursor_pos = None- for i, (file_path, seq_number, action_type, position) in enumerate(tqdm(screenshot_files, desc="Processing frames")):+ for turn_path, agent_response_path, screenshot_path in tqdm(turns, desc="Processing turns"):+ if not screenshot_path:+ continue # Skip turns without screenshots+# Load the current imagetry:- current_img = Image.open(file_path)+ current_img = Image.open(screenshot_path)except Exception as e:- print(f"Error loading image {file_path}: {e}")+ print(f"Error loading image {screenshot_path}: {e}")continue- # Current cursor position- current_cursor_pos = position+ # Extract action and position from agent response+ action_type = extract_action_from_agent_response(turn_path)+ current_cursor_pos = extract_cursor_position_from_agent_response(turn_path)++ # Extract thought from agent response+ current_thought = extract_thought_from_agent_response(turn_path)# Check if the current frame has an action (click/typing)is_action_frame = action_type in ["clicking", "typing"]@@ -535,9 +604,6 @@ def process_trajectory(trajectory_dir, output_dir, cursors):half_frames = FRAMES_PER_CLICK // 2# First half of animation uses PREVIOUS imagefor j in range(half_frames):- # Get the thought from the API response- current_thought = extract_thought_from_api_response(file_path)-output_img = create_cursor_overlay(prev_img, current_cursor_pos, cursors,thought_text=current_thought,@@ -552,9 +618,6 @@ def process_trajectory(trajectory_dir, output_dir, cursors):# Second half uses CURRENT imagefor j in range(half_frames, FRAMES_PER_CLICK):- # Get the thought from the API response- current_thought = extract_thought_from_api_response(file_path)-output_img = create_cursor_overlay(current_img, current_cursor_pos, cursors,thought_text=current_thought,@@ -569,9 +632,6 @@ def process_trajectory(trajectory_dir, output_dir, cursors):else:# If no previous frame, use current for full animationfor j in range(FRAMES_PER_CLICK):- # Get the thought from the API response- current_thought = extract_thought_from_api_response(file_path)-output_img = create_cursor_overlay(current_img, current_cursor_pos, cursors,thought_text=current_thought,@@ -585,9 +645,6 @@ def process_trajectory(trajectory_dir, output_dir, cursors):frame_index += 1else:# Regular frame with normal cursor- # Get the thought from the API response- current_thought = extract_thought_from_api_response(file_path)-output_img = create_cursor_overlay(current_img, current_cursor_pos, cursors,thought_text=current_thought,@@ -599,42 +656,43 @@ def process_trajectory(trajectory_dir, output_dir, cursors):output_img.save(os.path.join(output_dir, f"frame_{frame_index:04d}.png"))frame_index += 1- # Add position interpolation frames if we're not at the last frame- if i < len(screenshot_files) - 1:- # Get next position- next_cursor_pos = screenshot_files[i+1][3]-- # Only interpolate if both positions are valid and different- if current_cursor_pos is not None and next_cursor_pos is not None and current_cursor_pos != next_cursor_pos:- for j in range(1, FRAMES_PER_MOVE):- progress = j / FRAMES_PER_MOVE- interp_x = current_cursor_pos[0] + (next_cursor_pos[0] - current_cursor_pos[0]) * progress- interp_y = current_cursor_pos[1] + (next_cursor_pos[1] - current_cursor_pos[1]) * progress- interp_pos = (int(interp_x), int(interp_y))-- # Create interpolated movement frame- # Get the thought from the API response- current_thought = extract_thought_from_api_response(file_path)-- output_img = create_cursor_overlay(- current_img, interp_pos, cursors,- thought_text=current_thought,- cursor_type="normal",- frame_index=frame_index- )- # Apply animated vignette effect- output_img = create_animated_vignette(output_img, frame_index)- output_img.save(os.path.join(output_dir, f"frame_{frame_index:04d}.png"))- frame_index += 1-- # Save current frame as previous for next iteration+ # Store current frame as previous for next iterationprev_img = current_imgprev_cursor_pos = current_cursor_pos++ # Add position interpolation frames if we have both current and next turn data+ current_turn_index = turns.index((turn_path, agent_response_path, screenshot_path))+ if current_turn_index < len(turns) - 1:+ # Get next turn data+ next_turn_path, next_agent_response_path, next_screenshot_path = turns[current_turn_index + 1]+ if next_screenshot_path: # Only if next turn has a screenshot+ # Get next position+ next_cursor_pos = extract_cursor_position_from_agent_response(next_turn_path)++ # Only interpolate if both positions are valid and different+ if current_cursor_pos is not None and next_cursor_pos is not None and current_cursor_pos != next_cursor_pos:+ for j in range(1, FRAMES_PER_MOVE):+ progress = j / FRAMES_PER_MOVE+ interp_x = current_cursor_pos[0] + (next_cursor_pos[0] - current_cursor_pos[0]) * progress+ interp_y = current_cursor_pos[1] + (next_cursor_pos[1] - current_cursor_pos[1]) * progress+ interp_pos = (int(interp_x), int(interp_y))++ # Create interpolated movement frame+ output_img = create_cursor_overlay(+ current_img, interp_pos, cursors,+ thought_text=current_thought,+ cursor_type="normal",+ frame_index=frame_index+ )+ # Apply animated vignette effect+ output_img = create_animated_vignette(output_img, frame_index)+ output_img.save(os.path.join(output_dir, f"frame_{frame_index:04d}.png"))+ frame_index += 1def main():"""Main function to process the trajectory and create video frames."""parser = argparse.ArgumentParser(description='Create a video from a trajectory folder.')- parser.add_argument('trajectory_dir', type=str, help='Path to the trajectory folder')+ parser.add_argument('trajectory_dir', type=str, nargs='?', help='Path to the trajectory folder')parser.add_argument('--output_dir', type=str, default=OUTPUT_DIR, help='Output directory for video frames')parser.add_argument('--fps', type=int, default=24, help='Frames per second for the output video')parser.add_argument('--output_video', type=str, default='output_video.mp4', help='Path to output video file')@@ -642,6 +700,28 @@ def main():args = parser.parse_args()trajectory_dir = args.trajectory_dir++ # If trajectory_dir is not provided, find the latest folder in './trajectories'+ if trajectory_dir is None:+ trajectories_base_dir = "./trajectories"+ if os.path.exists(trajectories_base_dir) and os.path.isdir(trajectories_base_dir):+ # Get all directories in the trajectories folder+ trajectory_folders = [os.path.join(trajectories_base_dir, d) for d in os.listdir(trajectories_base_dir)+ if os.path.isdir(os.path.join(trajectories_base_dir, d))]++ if trajectory_folders:+ # Sort folders by modification time, most recent last+ trajectory_folders.sort(key=lambda x: os.path.getmtime(x))+ # Use the most recent folder+ trajectory_dir = trajectory_folders[-1]+ print(f"No trajectory directory specified, using latest: {trajectory_dir}")+ else:+ print(f"No trajectory folders found in {trajectories_base_dir}")+ return+ else:+ print(f"Trajectories directory {trajectories_base_dir} does not exist")+ return+output_dir = args.output_dirfps = args.fpsoutput_video = args.output_videodiff --git a/libs/agent/agent/providers/anthropic/loop.py b/libs/agent/agent/providers/anthropic/loop.pyindex 0ccdc79ad6..130a43cb26 100644--- a/libs/agent/agent/providers/anthropic/loop.py+++ b/libs/agent/agent/providers/anthropic/loop.py@@ -279,6 +279,8 @@ class AnthropicLoop(BaseLoop):messages,model=self.model,)+ # Log standardized response for ease of parsing+ self._log_api_call("agent_response", request=None, response=openai_compatible_response)await queue.put(openai_compatible_response)if not should_continue:diff --git a/libs/agent/agent/providers/omni/loop.py b/libs/agent/agent/providers/omni/loop.pyindex b53c120c68..18e0375f70 100644--- a/libs/agent/agent/providers/omni/loop.py+++ b/libs/agent/agent/providers/omni/loop.py@@ -670,6 +670,8 @@ class OmniLoop(BaseLoop):parsed_screen=parsed_screen,parser=self.parser)+ # Log standardized response for ease of parsing+ self._log_api_call("agent_response", request=None, response=openai_compatible_response)# Yield the response to the calleryield openai_compatible_responsediff --git a/libs/agent/agent/providers/openai/loop.py b/libs/agent/agent/providers/openai/loop.pyindex 8e507a1be1..c4e0dfb5db 100644--- a/libs/agent/agent/providers/openai/loop.py+++ b/libs/agent/agent/providers/openai/loop.py@@ -276,6 +276,10 @@ class OpenAILoop(BaseLoop):)# Don't reset last_response_id to None - keep the previous value if available++ # Log standardized response for ease of parsing+ # Since this is the openAI responses format, we don't need to convert it to agent response format+ self._log_api_call("agent_response", request=None, response=response)# Process API responseawait queue.put(response)diff --git a/libs/agent/agent/providers/uitars/loop.py b/libs/agent/agent/providers/uitars/loop.pyindex a30d3beeaf..ac14ed1e4f 100644--- a/libs/agent/agent/providers/uitars/loop.py+++ b/libs/agent/agent/providers/uitars/loop.py@@ -440,7 +440,7 @@ class UITARSLoop(BaseLoop):# MAIN LOOP - IMPLEMENTING ABSTRACT METHOD###########################################- async def run(self, messages: List[Dict[str, Any]]) -> AsyncGenerator[Dict[str, Any], None]:+ async def run(self, messages: List[Dict[str, Any]]) -> AsyncGenerator[AgentResponse, None]:"""Run the agent loop with provided messages.Args:@@ -513,6 +513,8 @@ class UITARSLoop(BaseLoop):messages,model=self.model,)+ # Log standardized response for ease of parsing+ self._log_api_call("agent_response", request=None, response=agent_response)yield agent_response# Check if we should continue this conversation