Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(benchmark): preserve native WPT harness timeouts

ldm0043185a1c4db
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/__main__.py b/moli-benchmark/moli_benchmark/wpt_cross/__main__.py
index e8e0442e9..6daea45d0 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/__main__.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/__main__.py
@@ -486,22 +486,15 @@ def _case_timeout(base_seconds: float, case: WptCase) -> float:
return base_seconds
-STEP_TIMEOUT_SENSITIVE_CASES = frozenset(
- {
- "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html",
- }
-)
+def _harness_timeout_multiplier(case: WptCase) -> float:
+ """Preserve WPT's own normal/long timeout semantics.
+ The benchmark's outer case deadline is transport and cleanup headroom. It
+ must not stretch testharness.js timers, or a test that should report a WPT
+ timeout can keep an engine busy until the outer deadline instead.
+ """
-def _harness_timeout_multiplier(
- case_timeout_seconds: float,
- default_harness_timeout_seconds: float,
- case_path: str | None = None,
-) -> float:
- case_path = case_path.split("?", 1)[0].split("#", 1)[0] if case_path else None
- if case_path in STEP_TIMEOUT_SENSITIVE_CASES:
- return 1.0
- return max(1.0, case_timeout_seconds / default_harness_timeout_seconds)
+ return max(1.0, case.timeout_multiplier)
def _is_layout_profile(profile: str) -> bool:
@@ -683,7 +676,7 @@ def main(argv: list[str] | None = None) -> int:
)
# Fixture server is started lazily (we need it before ANY engine runs).
- from .server import DEFAULT_TESTHARNESS_TIMEOUT_SECONDS, WptFixtureServer
+ from .server import WptFixtureServer
started_at = time.perf_counter()
engine_results: dict[str, list[dict[str, Any]]] = {}
@@ -724,17 +717,10 @@ def main(argv: list[str] | None = None) -> int:
engine_case_timeout = WPT_CROSS_CASE_TIMEOUT_SECONDS
server.set_harness_timeout_multipliers(
{
- case.case_path: _harness_timeout_multiplier(
- _case_timeout(engine_case_timeout, case),
- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
- case.case_path,
- )
+ case.case_path: _harness_timeout_multiplier(case)
for case in cases
},
- default_multiplier=_harness_timeout_multiplier(
- engine_case_timeout,
- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
- ),
+ default_multiplier=1.0,
)
cases_for_engine = [
_cdp_case_run(
@@ -776,11 +762,7 @@ def main(argv: list[str] | None = None) -> int:
case.case_path,
_url_for_case_origin(server, case.case_path, external=external),
_case_timeout(engine_case_timeout, case),
- _harness_timeout_multiplier(
- _case_timeout(engine_case_timeout, case),
- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
- case.case_path,
- ),
+ _harness_timeout_multiplier(case),
)
for case in cases
]
@@ -789,11 +771,7 @@ def main(argv: list[str] | None = None) -> int:
case.case_path,
_url_for_case_origin(server, case.case_path, external=external),
_case_timeout(engine_case_timeout, case),
- _harness_timeout_multiplier(
- _case_timeout(engine_case_timeout, case),
- DEFAULT_TESTHARNESS_TIMEOUT_SECONDS,
- case.case_path,
- ),
+ _harness_timeout_multiplier(case),
)
for case in scheduled_cases
]
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py b/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py
index 80f70ce46..a245c6cb6 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/cli_runner.py
@@ -35,7 +35,7 @@ from ..versions import sha256_file
from .engine import EngineDriver, build_driver
from .runner import CaseResult, EngineRunResult, classify_payload
from .runner import FINAL_PAYLOAD_SOURCES
-from .server import DEFAULT_TESTHARNESS_TIMEOUT_SECONDS, WptFixtureServer
+from .server import WptFixtureServer
CaseRun = tuple[str, str] | tuple[str, str, float] | tuple[str, str, float, float]
MAX_RECORDED_STDERR_CHARS = 2000
@@ -121,10 +121,6 @@ def _bridge_key_for_case(case_path: str) -> str:
return case_path if case_path.startswith("/") else "/" + case_path
-def _default_harness_timeout_multiplier(timeout_seconds: float) -> float:
- return max(1.0, timeout_seconds / DEFAULT_TESTHARNESS_TIMEOUT_SECONDS)
-
-
class _BenchPayloadParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
@@ -411,17 +407,12 @@ def run_engine_on_cases_cli(
return case[0], external, case[2], case[3]
if len(case) == 3:
timeout_seconds = case[2]
- return (
- case[0],
- external,
- timeout_seconds,
- _default_harness_timeout_multiplier(timeout_seconds),
- )
+ return (case[0], external, timeout_seconds, 1.0)
return (
case[0],
external,
case_timeout_seconds,
- _default_harness_timeout_multiplier(case_timeout_seconds),
+ 1.0,
)
results_by_path: dict[str, CaseResult] = {}
diff --git a/moli-benchmark/tests/test_wpt_cross.py b/moli-benchmark/tests/test_wpt_cross.py
index d886fe48f..51c3b5e4b 100644
--- a/moli-benchmark/tests/test_wpt_cross.py
+++ b/moli-benchmark/tests/test_wpt_cross.py
@@ -687,10 +687,14 @@ class WptCrossTests(unittest.TestCase):
],
)
- def test_harness_timeout_multiplier_never_shortens_wpt_delays(self) -> None:
- self.assertEqual(_harness_timeout_multiplier(8.0, 10.0), 1.0)
- self.assertEqual(_harness_timeout_multiplier(10.0, 10.0), 1.0)
- self.assertEqual(_harness_timeout_multiplier(50.0, 10.0), 5.0)
+ def test_harness_timeout_multiplier_uses_wpt_case_metadata(self) -> None:
+ self.assertEqual(_harness_timeout_multiplier(WptCase("normal.html")), 1.0)
+ self.assertEqual(
+ _harness_timeout_multiplier(
+ WptCase("long.html", timeout_multiplier=LONG_TIMEOUT_MULTIPLIER)
+ ),
+ LONG_TIMEOUT_MULTIPLIER,
+ )
def test_lightpanda_cli_fetch_keeps_configured_timeout(self) -> None:
command = _lightpanda_fetch(Path("/bin/lightpanda"), "http://127.0.0.1:8000/case.html", 30.0)
@@ -1606,19 +1610,19 @@ class WptCrossTests(unittest.TestCase):
"secure-contexts/basic-shared-worker.html",
"http://[2001:db8::1]:9000/secure-contexts/basic-shared-worker.html",
WPT_CROSS_CASE_TIMEOUT_SECONDS,
- WPT_CROSS_CASE_TIMEOUT_SECONDS / 10.0,
+ 1.0,
),
(
"digital-credentials/non-secure-contexts.http.html",
"http://[2001:db8::1]:9000/digital-credentials/non-secure-contexts.http.html",
WPT_CROSS_CASE_TIMEOUT_SECONDS,
- WPT_CROSS_CASE_TIMEOUT_SECONDS / 10.0,
+ 1.0,
),
(
"WebCryptoAPI/digest/digest.https.html",
"http://127.0.0.1:8000/WebCryptoAPI/digest/digest.https.html",
WPT_CROSS_CASE_TIMEOUT_SECONDS,
- WPT_CROSS_CASE_TIMEOUT_SECONDS / 10.0,
+ 1.0,
)
]
],
@@ -1952,7 +1956,7 @@ class WptCrossTests(unittest.TestCase):
self.assertEqual(summary["profile"], "layout-testharness")
self.assertEqual(summary["viewport"]["width"], 800)
- def test_main_preserves_harness_multiplier_except_step_timeout_sensitive_cases(self) -> None:
+ def test_main_uses_wpt_harness_multipliers_independent_of_outer_deadline(self) -> None:
calls = []
class FakeServer:
@@ -2045,21 +2049,21 @@ class WptCrossTests(unittest.TestCase):
[
(
{
- "normal.html": 12.0,
- "long.html": 12.0,
+ "normal.html": 1.0,
+ "long.html": LONG_TIMEOUT_MULTIPLIER,
"html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html": 1.0,
},
- 12.0,
+ 1.0,
)
],
)
- def test_harness_timeout_multiplier_keeps_dynamic_import_step_timeout_unscaled(self) -> None:
+ def test_normal_dynamic_import_uses_normal_wpt_timeout(self) -> None:
case = WptCase(
"html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html"
)
- self.assertEqual(_harness_timeout_multiplier(30.0, 10.0, case.case_path), 1.0)
+ self.assertEqual(_harness_timeout_multiplier(case), 1.0)
def test_parser_rejects_removed_timeout_flags_and_legacy_parallelism(self) -> None:
parser = _build_parser()
@@ -5081,7 +5085,7 @@ test(() => {}, "ok");
self.assertIsNone(result.error)
self.assertEqual(result.payload_source, "completion-callback")
- def test_cli_runner_uses_process_pool_even_for_single_parallelism(self) -> None:
+ def test_cli_runner_defaults_harness_timeout_independent_of_process_deadline(self) -> None:
class FakeFuture:
def __init__(self, result: CaseResult) -> None:
self._result = result
@@ -5151,11 +5155,16 @@ test(() => {}, "ok");
fixture_server=fixture_server,
cases=[
(
- "case.html",
- "http://[2001:db8::1]:9000/case.html",
- 12.0,
- 1.2,
- )
+ "normal.html",
+ "http://[2001:db8::1]:9000/normal.html",
+ 120.0,
+ ),
+ (
+ "long.html",
+ "http://[2001:db8::1]:9000/long.html",
+ 120.0,
+ LONG_TIMEOUT_MULTIPLIER,
+ ),
],
parallelism=1,
progress_every=0,
@@ -5164,11 +5173,15 @@ test(() => {}, "ok");
pool = FakeProcessPoolExecutor.instances[0]
self.assertEqual(pool.max_workers, 1)
self.assertEqual(pool.mp_context.get_start_method(), "spawn")
- self.assertEqual(pool.submitted[0][1].case_path, "case.html")
+ self.assertEqual(pool.submitted[0][1].case_path, "normal.html")
self.assertTrue(pool.submitted[0][1].external)
- self.assertEqual(pool.submitted[0][1].timeout_seconds, 12.0)
- self.assertEqual(pool.submitted[0][1].harness_timeout_multiplier, 1.2)
- self.assertEqual(result.cases[0].status, "pass")
+ self.assertEqual(pool.submitted[0][1].timeout_seconds, 120.0)
+ self.assertEqual(pool.submitted[0][1].harness_timeout_multiplier, 1.0)
+ self.assertEqual(
+ pool.submitted[1][1].harness_timeout_multiplier,
+ LONG_TIMEOUT_MULTIPLIER,
+ )
+ self.assertTrue(all(case.status == "pass" for case in result.cases))
self.assertEqual(result.shutdown_info["scheduler"], "process-pool")
def test_moli_cli_worker_appends_hardcoded_wpt_user_agent(self) -> None: