Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added results table
Dillon DuPontc2302eb6c607
diff --git a/examples/eval_examples.py b/examples/eval_examples.pyindex 1978a8976f..b3d163ca0c 100644--- a/examples/eval_examples.py+++ b/examples/eval_examples.py@@ -11,13 +11,16 @@ import urllib.requestimport datetimefrom urllib.parse import quote+# Global variable to track all results+all_results = []+# Wikirace prompt templateWIKIRACE_PROMPT_TEMPLATE = """You are playing Wikirace in {browser}! Your goal is to navigate from "{start_page}" to "{target_page}"by clicking only on Wikipedia links within articles.Rules:-1. Only click on links within Wikipedia articles (blue underlined text)+1. Only click on links within Wikipedia articles (blue text)2. No using search, back button, or typing URLs3. You MAY use cmd+f (or ctrl+f) to find text on the current page4. Do NOT click any search icon or type into any search box unless it's a browser command@@ -26,6 +29,7 @@ Rules:7. Do not maximize the window or use any other application8. Avoid wasting actions by scrolling9. Try using cmd+f and quickly clicking through relevant links in the page as you have a limited number of steps+10. Stay on the English WikipediaLook at the current page and click on a link that might lead you closer to {target_page}."""@@ -36,6 +40,7 @@ _print = print# Define log file pathproject_root = Path(__file__).parent.parentlog_file = project_root / "examples" / "evals" / "eval_appuse_log.txt"+results_file = project_root / "examples" / "evals" / "eval_appuse_results.md"# Custom print function that also logs to filedef print(*args, **kwargs):@@ -160,6 +165,36 @@ def get_article_pair(depth=5):target_article = wikipedia_random_walk(start_article, depth)[-1]return start_article, target_article++def save_results_to_markdown():+ """Save all results to a markdown table"""+ global all_results++ if not all_results:+ print("No results to save")+ return++ # Create header for the markdown table+ header = "| Timestamp | Scenario | App-Use | Browser | Config | Start | Target | Steps | Success | Duration (s) |"+ separator = "|---|---|---|---|---|---|---|---|---|---|"++ # Create rows for each result+ rows = []+ for result in all_results:+ timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+ row = f"| {timestamp} | {result['scenario']} | {result['app_use']} | {result['browser']} | {result['config']} | {result['start']} | {result['target']} | {result['steps']} | {result['success']} | {result['duration']:.2f} |"+ rows.append(row)++ # Combine header, separator, and rows+ table = "\n".join([header, separator] + rows)++ # Write to file (append mode)+ with open(results_file, "a") as f:+ f.write(f"\n\n## Results Update - {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")+ f.write(table)++ print(f"Results saved to {results_file}")+async def run_scenario(scenario_name, use_app_use, agent_configs, max_steps=30):"""Run a specific evaluation scenario"""@@ -254,6 +289,7 @@ async def open_wiki(page, app_name="Safari"):async def run_messy_desktop_scenario(computer, agent_configs, max_steps):+ global all_results"""Run the messy desktop scenario with a single agent"""# Get popular wiki articlesglobal articles@@ -292,7 +328,8 @@ async def run_messy_desktop_scenario(computer, agent_configs, max_steps):agent = ComputerAgent(computer=agent_computer,loop=loop_provider,- model=LLM(model_provider) if not isinstance(model_provider, LLM) else model_provider+ model=LLM(model_provider) if not isinstance(model_provider, LLM) else model_provider,+ trajectory_dir="examples/evals/trajectories/eval_appuse")# Run the wikirace@@ -387,6 +424,7 @@ async def run_messy_desktop_scenario(computer, agent_configs, max_steps):async def run_parallel_agents_scenario(computer, agent_configs, max_steps):+ global all_results"""Run two agents in parallel, one using Safari and one using Firefox"""# Get popular wiki articles@@ -436,13 +474,15 @@ async def run_parallel_agents_scenario(computer, agent_configs, max_steps):safari_agent = ComputerAgent(computer=safari_desktop,loop=loop_provider,- model=LLM(model_provider) if not isinstance(model_provider, LLM) else model_provider+ model=LLM(model_provider) if not isinstance(model_provider, LLM) else model_provider,+ trajectory_dir="examples/evals/trajectories/eval_parallel_safari")firefox_agent = ComputerAgent(computer=firefox_desktop,loop=loop_provider,- model=LLM(model_provider) if not isinstance(model_provider, LLM) else model_provider+ model=LLM(model_provider) if not isinstance(model_provider, LLM) else model_provider,+ trajectory_dir="examples/evals/trajectories/eval_parallel_firefox")# Create prompts using the template@@ -525,6 +565,24 @@ async def run_parallel_agents_scenario(computer, agent_configs, max_steps):print(f"{browser} current page: {current_page}")print(f"{browser} target: {target_page}")+ # Add result to global tracking+ global all_results+ current_result = {+ 'scenario': 'parallel_agents',+ 'app_use': 'Yes' if 'app-use' in (computer.experiments or []) else 'No',+ 'browser': browser,+ 'config': config_name,+ 'start': start_page,+ 'target': target_page,+ 'steps': results['steps'],+ 'success': results['success'],+ 'duration': time.time() - results['start_time']+ }+ all_results.append(current_result)++ # Save results after each step+ save_results_to_markdown()+# Check if we reached the targetif current_page and target_page.lower() in current_page.lower():results["success"] = True@@ -575,9 +633,9 @@ async def main():# Define agent configurations to testagent_configs = [- ("OpenAI", AgentLoop.OPENAI, LLMProvider.OPENAI),- ("Anthropic", AgentLoop.ANTHROPIC, LLMProvider.ANTHROPIC),- # ("UITARS", AgentLoop.UITARS, LLM(LLMProvider.OAICOMPAT, name="tgi", provider_base_url=os.getenv("UITARS_BASE_URL")))+ # ("OpenAI", AgentLoop.OPENAI, LLMProvider.OPENAI),+ # ("Anthropic", AgentLoop.ANTHROPIC, LLMProvider.ANTHROPIC),+ ("UITARS", AgentLoop.UITARS, LLM(LLMProvider.OAICOMPAT, name="tgi", provider_base_url=os.getenv("UITARS_BASE_URL")))]# # Run the test scenario without any agents@@ -585,7 +643,7 @@ async def main():# await run_test_scenario()# Set maximum steps for each agent run- max_steps = 50+ max_steps = 15runs = 5# run all scenarios