Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(benchmark): preserve native WPT harness timeouts
ldm0043185a1c4db
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/__main__.py b/moli-benchmark/moli_benchmark/wpt_cross/__main__.pyindex e8e0442e9..6daea45d0 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/__main__.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/__main__.py@@ -486,22 +486,15 @@ def _case_timeout(base_seconds: float, case: WptCase) -> float:return base_seconds-STEP_TIMEOUT_SENSITIVE_CASES = frozenset(- {- "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html",- }-)+def _harness_timeout_multiplier(case: WptCase) -> float:+ """Preserve WPT's own normal/long timeout semantics.+ The benchmark's outer case deadline is transport and cleanup headroom. It+ must not stretch testharness.js timers, or a test that should report a WPT+ timeout can keep an engine busy until the outer deadline instead.+ """-def _harness_timeout_multiplier(- case_timeout_seconds: float,- default_harness_timeout_seconds: float,- case_path: str | None = None,-) -> float:- case_path = case_path.split("?", 1)[0].split("#", 1)[0] if case_path else None- if case_path in STEP_TIMEOUT_SENSITIVE_CASES:- return 1.0- return max(1.0, case_timeout_seconds / default_harness_timeout_seconds)+ return max(1.0, case.timeout_multiplier)def _is_layout_profile(profile: str) -> bool:@@ -683,7 +676,7 @@ def main(argv: list[str] | None = None) -> int:)# Fixture server is started lazily (we need it before ANY engine runs).- from .server import DEFAULT_TESTHARNESS_TIMEOUT_SECONDS, WptFixtureServer+ from .server import WptFixtureServerstarted_at = time.perf_counter()engine_results: dict[str, list[dict[str, Any]]] = {}@@ -724,17 +717,10 @@ def main(argv: list[str] | None = None) -> int:engine_case_timeout = WPT_CROSS_CASE_TIMEOUT_SECONDSserver.set_harness_timeout_multipliers({- case.case_path: _harness_timeout_multiplier(- _case_timeout(engine_case_timeout, case),- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,- case.case_path,- )+ case.case_path: _harness_timeout_multiplier(case)for case in cases},- default_multiplier=_harness_timeout_multiplier(- engine_case_timeout,- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,- ),+ default_multiplier=1.0,)cases_for_engine = [_cdp_case_run(@@ -776,11 +762,7 @@ def main(argv: list[str] | None = None) -> int:case.case_path,_url_for_case_origin(server, case.case_path, external=external),_case_timeout(engine_case_timeout, case),- _harness_timeout_multiplier(- _case_timeout(engine_case_timeout, case),- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,- case.case_path,- ),+ _harness_timeout_multiplier(case),)for case in cases]@@ -789,11 +771,7 @@ def main(argv: list[str] | None = None) -> int:case.case_path,_url_for_case_origin(server, case.case_path, external=external),_case_timeout(engine_case_timeout, case),- _harness_timeout_multiplier(- _case_timeout(engine_case_timeout, case),- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,- case.case_path,- ),+ _harness_timeout_multiplier(case),)for case in scheduled_cases]diff --git a/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py b/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.pyindex 80f70ce46..a245c6cb6 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py@@ -35,7 +35,7 @@ from ..versions import sha256_filefrom .engine import EngineDriver, build_driverfrom .runner import CaseResult, EngineRunResult, classify_payloadfrom .runner import FINAL_PAYLOAD_SOURCES-from .server import DEFAULT_TESTHARNESS_TIMEOUT_SECONDS, WptFixtureServer+from .server import WptFixtureServerCaseRun = tuple[str, str] | tuple[str, str, float] | tuple[str, str, float, float]MAX_RECORDED_STDERR_CHARS = 2000@@ -121,10 +121,6 @@ def _bridge_key_for_case(case_path: str) -> str:return case_path if case_path.startswith("/") else "/" + case_path-def _default_harness_timeout_multiplier(timeout_seconds: float) -> float:- return max(1.0, timeout_seconds / DEFAULT_TESTHARNESS_TIMEOUT_SECONDS)--class _BenchPayloadParser(HTMLParser):def __init__(self) -> None:super().__init__(convert_charrefs=True)@@ -411,17 +407,12 @@ def run_engine_on_cases_cli(return case[0], external, case[2], case[3]if len(case) == 3:timeout_seconds = case[2]- return (- case[0],- external,- timeout_seconds,- _default_harness_timeout_multiplier(timeout_seconds),- )+ return (case[0], external, timeout_seconds, 1.0)return (case[0],external,case_timeout_seconds,- _default_harness_timeout_multiplier(case_timeout_seconds),+ 1.0,)results_by_path: dict[str, CaseResult] = {}diff --git a/moli-benchmark/tests/test_wpt_cross.py b/moli-benchmark/tests/test_wpt_cross.pyindex d886fe48f..51c3b5e4b 100644--- a/moli-benchmark/tests/test_wpt_cross.py+++ b/moli-benchmark/tests/test_wpt_cross.py@@ -687,10 +687,14 @@ class WptCrossTests(unittest.TestCase):],)- def test_harness_timeout_multiplier_never_shortens_wpt_delays(self) -> None:- self.assertEqual(_harness_timeout_multiplier(8.0, 10.0), 1.0)- self.assertEqual(_harness_timeout_multiplier(10.0, 10.0), 1.0)- self.assertEqual(_harness_timeout_multiplier(50.0, 10.0), 5.0)+ def test_harness_timeout_multiplier_uses_wpt_case_metadata(self) -> None:+ self.assertEqual(_harness_timeout_multiplier(WptCase("normal.html")), 1.0)+ self.assertEqual(+ _harness_timeout_multiplier(+ WptCase("long.html", timeout_multiplier=LONG_TIMEOUT_MULTIPLIER)+ ),+ LONG_TIMEOUT_MULTIPLIER,+ )def test_lightpanda_cli_fetch_keeps_configured_timeout(self) -> None:command = _lightpanda_fetch(Path("/bin/lightpanda"), "http://127.0.0.1:8000/case.html", 30.0)@@ -1606,19 +1610,19 @@ class WptCrossTests(unittest.TestCase):"secure-contexts/basic-shared-worker.html","http://[2001:db8::1]:9000/secure-contexts/basic-shared-worker.html",WPT_CROSS_CASE_TIMEOUT_SECONDS,- WPT_CROSS_CASE_TIMEOUT_SECONDS / 10.0,+ 1.0,),("digital-credentials/non-secure-contexts.http.html","http://[2001:db8::1]:9000/digital-credentials/non-secure-contexts.http.html",WPT_CROSS_CASE_TIMEOUT_SECONDS,- WPT_CROSS_CASE_TIMEOUT_SECONDS / 10.0,+ 1.0,),("WebCryptoAPI/digest/digest.https.html","http://127.0.0.1:8000/WebCryptoAPI/digest/digest.https.html",WPT_CROSS_CASE_TIMEOUT_SECONDS,- WPT_CROSS_CASE_TIMEOUT_SECONDS / 10.0,+ 1.0,)]],@@ -1952,7 +1956,7 @@ class WptCrossTests(unittest.TestCase):self.assertEqual(summary["profile"], "layout-testharness")self.assertEqual(summary["viewport"]["width"], 800)- def test_main_preserves_harness_multiplier_except_step_timeout_sensitive_cases(self) -> None:+ def test_main_uses_wpt_harness_multipliers_independent_of_outer_deadline(self) -> None:calls = []class FakeServer:@@ -2045,21 +2049,21 @@ class WptCrossTests(unittest.TestCase):[({- "normal.html": 12.0,- "long.html": 12.0,+ "normal.html": 1.0,+ "long.html": LONG_TIMEOUT_MULTIPLIER,"html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html": 1.0,},- 12.0,+ 1.0,)],)- def test_harness_timeout_multiplier_keeps_dynamic_import_step_timeout_unscaled(self) -> None:+ def test_normal_dynamic_import_uses_normal_wpt_timeout(self) -> None:case = WptCase("html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html")- self.assertEqual(_harness_timeout_multiplier(30.0, 10.0, case.case_path), 1.0)+ self.assertEqual(_harness_timeout_multiplier(case), 1.0)def test_parser_rejects_removed_timeout_flags_and_legacy_parallelism(self) -> None:parser = _build_parser()@@ -5081,7 +5085,7 @@ test(() => {}, "ok");self.assertIsNone(result.error)self.assertEqual(result.payload_source, "completion-callback")- def test_cli_runner_uses_process_pool_even_for_single_parallelism(self) -> None:+ def test_cli_runner_defaults_harness_timeout_independent_of_process_deadline(self) -> None:class FakeFuture:def __init__(self, result: CaseResult) -> None:self._result = result@@ -5151,11 +5155,16 @@ test(() => {}, "ok");fixture_server=fixture_server,cases=[(- "case.html",- "http://[2001:db8::1]:9000/case.html",- 12.0,- 1.2,- )+ "normal.html",+ "http://[2001:db8::1]:9000/normal.html",+ 120.0,+ ),+ (+ "long.html",+ "http://[2001:db8::1]:9000/long.html",+ 120.0,+ LONG_TIMEOUT_MULTIPLIER,+ ),],parallelism=1,progress_every=0,@@ -5164,11 +5173,15 @@ test(() => {}, "ok");pool = FakeProcessPoolExecutor.instances[0]self.assertEqual(pool.max_workers, 1)self.assertEqual(pool.mp_context.get_start_method(), "spawn")- self.assertEqual(pool.submitted[0][1].case_path, "case.html")+ self.assertEqual(pool.submitted[0][1].case_path, "normal.html")self.assertTrue(pool.submitted[0][1].external)- self.assertEqual(pool.submitted[0][1].timeout_seconds, 12.0)- self.assertEqual(pool.submitted[0][1].harness_timeout_multiplier, 1.2)- self.assertEqual(result.cases[0].status, "pass")+ self.assertEqual(pool.submitted[0][1].timeout_seconds, 120.0)+ self.assertEqual(pool.submitted[0][1].harness_timeout_multiplier, 1.0)+ self.assertEqual(+ pool.submitted[1][1].harness_timeout_multiplier,+ LONG_TIMEOUT_MULTIPLIER,+ )+ self.assertTrue(all(case.status == "pass" for case in result.cases))self.assertEqual(result.shutdown_info["scheduler"], "process-pool")def test_moli_cli_worker_appends_hardcoded_wpt_user_agent(self) -> None: