Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
host html at localhost instead of about:blank
Dillon DuPontdaa68874a914
diff --git a/libs/python/bench-ui/bench_ui/child.py b/libs/python/bench-ui/bench_ui/child.pyindex b345b68eec..28eff3904e 100644--- a/libs/python/bench-ui/bench_ui/child.py+++ b/libs/python/bench-ui/bench_ui/child.py@@ -18,7 +18,7 @@ def _get_free_port() -> int:return s.getsockname()[1]-def _start_http_server(window: webview.Window, port: int, ready_event: threading.Event):+def _start_http_server(window: webview.Window, port: int, ready_event: threading.Event, html_content: str | None = None):async def rect_handler(request: web.Request):try:data = await request.json()@@ -90,7 +90,13 @@ def _start_http_server(window: webview.Window, port: int, ready_event: threadingreturn web.json_response({"error": str(e)}, status=500)return web.json_response({"result": result})+ async def index_handler(request: web.Request):+ if html_content is None:+ return web.json_response({"status": "ok", "message": "bench-ui control server"})+ return web.Response(text=html_content, content_type="text/html")+app = web.Application()+ app.router.add_get("/", index_handler)app.router.add_post("/rect", rect_handler)app.router.add_post("/eval", eval_handler)@@ -127,6 +133,9 @@ def main():use_inner_size: bool = bool(cfg.get("use_inner_size", False))title_bar_style: str = cfg.get("title_bar_style", "default")+ # Choose port early so we can point the window to it when serving inline HTML+ port = _get_free_port()+# Create windowif url:window = webview.create_window(@@ -140,10 +149,13 @@ def main():text_select=True,background_color="#FFFFFF",)+ html_for_server = Noneelse:+ # Serve inline HTML at control server root and point window to it+ resolved_url = f"http://127.0.0.1:{port}/"window = webview.create_window(title,- html=html,+ url=resolved_url,width=width,height=height,x=x,@@ -152,6 +164,7 @@ def main():text_select=True,background_color="#FFFFFF",)+ html_for_server = html# Track when the page is loaded so JS execution succeedswindow_ready = threading.Event()@@ -159,9 +172,8 @@ def main():window_ready.set()window.events.loaded += _on_loaded # type: ignore[attr-defined]- # Start HTTP server for control- port = _get_free_port()- _start_http_server(window, port, window_ready)+ # Start HTTP server for control (and optionally serve inline HTML)+ _start_http_server(window, port, window_ready, html_content=html_for_server)# Print startup info for parent to readprint(json.dumps({"pid": os.getpid(), "port": port}), flush=True)diff --git a/libs/python/bench-ui/examples/output_overlay.png b/libs/python/bench-ui/examples/output_overlay.pngindex d7e3a493b3..1ae98a25d0 100644Binary files a/libs/python/bench-ui/examples/output_overlay.png and b/libs/python/bench-ui/examples/output_overlay.png differdiff --git a/libs/python/bench-ui/examples/simple_example.py b/libs/python/bench-ui/examples/simple_example.pyindex 6a99abede0..bd703f20b0 100644--- a/libs/python/bench-ui/examples/simple_example.py+++ b/libs/python/bench-ui/examples/simple_example.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport timefrom bench_ui import launch_window, get_element_rect, execute_javascriptfrom pathlib import Path+import osHTML = """<!doctype html>@@ -17,11 +18,25 @@ HTML = """<body><h1>Bench UI Example</h1><div id="target">Hello from pywebview</div>+++ <h1>Click the button</h1>+ <button id="submit" class="btn" data-instruction="the button">Submit</button>+ <script>+ window.__submitted = false;+ document.getElementById('submit').addEventListener('click', function() {+ window.__submitted = true;+ this.textContent = 'Submitted!';+ this.disabled = true;+ });+ </script></body></html>"""def main():+ os.environ["CUA_BENCH_UI_DEBUG"] = "1"+# Launch a window with inline HTML contentpid = launch_window(html=HTML,@@ -54,7 +69,7 @@ def main():print(f"Failed to capture/annotate screenshot: {e}")# Execute arbitrary JavaScript- text = execute_javascript(pid, "document.querySelector('#t')?.textContent")+ text = execute_javascript(pid, "window.__submitted")print("text:", text)diff --git a/libs/python/bench-ui/pyproject.toml b/libs/python/bench-ui/pyproject.tomlindex 101d9a31dc..daae6f8289 100644--- a/libs/python/bench-ui/pyproject.toml+++ b/libs/python/bench-ui/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "pdm.backend"[project]name = "cua-bench-ui"-version = "0.4.0"+version = "0.6.0"description = "Lightweight webUI window controller for CUA bench using pywebview"readme = "README.md"authors = [