Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

host html at localhost instead of about:blank

Dillon DuPontdaa68874a914
diff --git a/libs/python/bench-ui/bench_ui/child.py b/libs/python/bench-ui/bench_ui/child.py
index b345b68eec..28eff3904e 100644
--- a/libs/python/bench-ui/bench_ui/child.py
+++ b/libs/python/bench-ui/bench_ui/child.py
@@ -18,7 +18,7 @@ def _get_free_port() -> int:
return s.getsockname()[1]
-def _start_http_server(window: webview.Window, port: int, ready_event: threading.Event):
+def _start_http_server(window: webview.Window, port: int, ready_event: threading.Event, html_content: str | None = None):
async def rect_handler(request: web.Request):
try:
data = await request.json()
@@ -90,7 +90,13 @@ def _start_http_server(window: webview.Window, port: int, ready_event: threading
return web.json_response({"error": str(e)}, status=500)
return web.json_response({"result": result})
+ async def index_handler(request: web.Request):
+ if html_content is None:
+ return web.json_response({"status": "ok", "message": "bench-ui control server"})
+ return web.Response(text=html_content, content_type="text/html")
+
app = web.Application()
+ app.router.add_get("/", index_handler)
app.router.add_post("/rect", rect_handler)
app.router.add_post("/eval", eval_handler)
@@ -127,6 +133,9 @@ def main():
use_inner_size: bool = bool(cfg.get("use_inner_size", False))
title_bar_style: str = cfg.get("title_bar_style", "default")
+ # Choose port early so we can point the window to it when serving inline HTML
+ port = _get_free_port()
+
# Create window
if url:
window = webview.create_window(
@@ -140,10 +149,13 @@ def main():
text_select=True,
background_color="#FFFFFF",
)
+ html_for_server = None
else:
+ # Serve inline HTML at control server root and point window to it
+ resolved_url = f"http://127.0.0.1:{port}/"
window = webview.create_window(
title,
- html=html,
+ url=resolved_url,
width=width,
height=height,
x=x,
@@ -152,6 +164,7 @@ def main():
text_select=True,
background_color="#FFFFFF",
)
+ html_for_server = html
# Track when the page is loaded so JS execution succeeds
window_ready = threading.Event()
@@ -159,9 +172,8 @@ def main():
window_ready.set()
window.events.loaded += _on_loaded # type: ignore[attr-defined]
- # Start HTTP server for control
- port = _get_free_port()
- _start_http_server(window, port, window_ready)
+ # Start HTTP server for control (and optionally serve inline HTML)
+ _start_http_server(window, port, window_ready, html_content=html_for_server)
# Print startup info for parent to read
print(json.dumps({"pid": os.getpid(), "port": port}), flush=True)
diff --git a/libs/python/bench-ui/examples/output_overlay.png b/libs/python/bench-ui/examples/output_overlay.png
index d7e3a493b3..1ae98a25d0 100644
Binary files a/libs/python/bench-ui/examples/output_overlay.png and b/libs/python/bench-ui/examples/output_overlay.png differ
diff --git a/libs/python/bench-ui/examples/simple_example.py b/libs/python/bench-ui/examples/simple_example.py
index 6a99abede0..bd703f20b0 100644
--- a/libs/python/bench-ui/examples/simple_example.py
+++ b/libs/python/bench-ui/examples/simple_example.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import time
from bench_ui import launch_window, get_element_rect, execute_javascript
from pathlib import Path
+import os
HTML = """
<!doctype html>
@@ -17,11 +18,25 @@ HTML = """
<body>
<h1>Bench UI Example</h1>
<div id="target">Hello from pywebview</div>
+
+
+ <h1>Click the button</h1>
+ <button id="submit" class="btn" data-instruction="the button">Submit</button>
+ <script>
+ window.__submitted = false;
+ document.getElementById('submit').addEventListener('click', function() {
+ window.__submitted = true;
+ this.textContent = 'Submitted!';
+ this.disabled = true;
+ });
+ </script>
</body>
</html>
"""
def main():
+ os.environ["CUA_BENCH_UI_DEBUG"] = "1"
+
# Launch a window with inline HTML content
pid = launch_window(
html=HTML,
@@ -54,7 +69,7 @@ def main():
print(f"Failed to capture/annotate screenshot: {e}")
# Execute arbitrary JavaScript
- text = execute_javascript(pid, "document.querySelector('#t')?.textContent")
+ text = execute_javascript(pid, "window.__submitted")
print("text:", text)
diff --git a/libs/python/bench-ui/pyproject.toml b/libs/python/bench-ui/pyproject.toml
index 101d9a31dc..daae6f8289 100644
--- a/libs/python/bench-ui/pyproject.toml
+++ b/libs/python/bench-ui/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
[project]
name = "cua-bench-ui"
-version = "0.4.0"
+version = "0.6.0"
description = "Lightweight webUI window controller for CUA bench using pywebview"
readme = "README.md"
authors = [