Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Split WPT cross tests by responsibility
ldm0d299588b68aa
diff --git a/moli-benchmark/tests/test_wpt_cross.py b/moli-benchmark/tests/test_wpt_cross.pyindex ce759332e..e9eeb9b0c 100644--- a/moli-benchmark/tests/test_wpt_cross.py+++ b/moli-benchmark/tests/test_wpt_cross.py@@ -1,159 +1,9 @@from __future__ import annotations-import asyncio-import hashlib-import io-import json-import socket-import subprocess-import tempfile-import threading-import time-import unittest-from contextlib import redirect_stderr, redirect_stdout-from http.client import HTTPConnection-from io import StringIO-from pathlib import Path-from types import SimpleNamespace-from unittest.mock import AsyncMock, Mock, patch-from urllib.request import Request, urlopen+from wpt_cross_test_support import *-from PIL import Image-from moli_benchmark.config import clear_current_proxy_env-from moli_benchmark.raw_cdp import RawCdpError-from moli_benchmark.wpt_cross.__main__ import (- CASE_LIST_FILES,- WPT_CROSS_CASE_TIMEOUT_SECONDS,- WPT_CROSS_PARALLELISM,- _build_parser,- _case_requires_non_trustworthy_origin,- _case_requires_trustworthy_origin,- _deduplicate_cases,- _harness_timeout_multiplier,- _is_full_case_list_run,- _recorded_failure_drift,- _url_for_case_origin,- _write_repo_case_lists,- main,-)-from moli_benchmark.wpt_cross.build_partial import main as build_partial_main-from moli_benchmark.wpt_cross.case_set import (- ANY_JS_WINDOW_QUERY,- WINDOW_JS_WINDOW_QUERY,- any_js_window_case_path,- DEFAULT_EXCLUDE_DIR_PREFIXES,- FuzzyTolerance,- LAYOUT_PROFILE_DIR_PREFIXES,- LONG_TIMEOUT_MULTIPLIER,- ReftestReference,- WptCase,- enumerate_cases,- enumerate_reftest_cases,- explicit_reftest_case,- explicit_case,- parse_any_js_meta,- window_js_window_case_path,-)-from moli_benchmark.wpt_cross.engine import (- _moli_command,- _moli_fetch,- _lightpanda_fetch,-)-from moli_benchmark.wpt_cross.cli_runner import (- MOLI_WPT_USER_AGENT,- _CliCaseWorkerInput,- _CliSubprocessResult,- _classify_cli_case_result,- _moli_fixture_host_resolve_args,- _nonzero_exit_status,- _payload_from_stdout_html,- _payload_grace_for_process_result,- _run_cli_case_worker,- _stderr_tail,- run_engine_on_cases_cli,-)-from moli_benchmark.wpt_cross.any_js import (- ANY_JS_DEDICATED_WORKER_GLOBAL,- ANY_JS_WINDOW_GLOBAL,- any_js_case_path_for_global,- any_js_source_script_path,- any_js_worker_script_path,-)-from moli_benchmark.wpt_cross.render_html import render_html-from moli_benchmark.wpt_cross.runner import (- _AttachedPage,- _PageSessionUnusable,- _ReftestEvidence,- _close_page,- _has_navigation_evidence,- _navigation_identity,- _normalized_case_path,- _normalized_navigation_url,- _run_async,- _run_one_case,- _write_reftest_failure_artifacts,- CapturedScreenshot,- CaseResult,- EngineRunResult,- LAYOUT_VIEWPORT,- ReftestReferenceRun,- ReftestRun,- case_result_to_dict,- classify_payload,- compare_reftest_screenshots,- reftest_comparisons_pass,- reftest_relation_passes,-)-from moli_benchmark.wpt_cross.scheduler import (- FIXED_RUN_SHUFFLE_SEED,- build_run_schedule,-)-from moli_benchmark.wpt_cross.server import (- _fetch_status_response,- BENCH_REPORT_BRIDGE,- BENCH_TESTDRIVER_VENDOR_BRIDGE,- BENCH_TIMEOUT_MULTIPLIER_QUERY,- WptFixtureServer,- ResultsStore,- _apply_header_operations,- _any_js_window_wrapper,- _bench_report_bridge,- _content_security_policy_resource_response,- _workers_modules_export_on_load_script_response,- _inject_bench_report_bridge_config,- _host_header_hostname,- _headers_include,- _normalize_harness_case_key,- _needs_wpt_template_substitution,- _legacy_wpt_resource_alias,- _pipe_response_header_operations,- _pipe_response_status,- _pipe_trickle_delay_seconds,- _redirect_fixture_response,- _response_content_type_and_extra_headers,- _resolve_wpt_static_script_url,- _sidecar_response_headers,- _static_response_header_block,- _static_response_headers,- _substitute_wpt_template_variables,- _window_js_window_wrapper,- _wasm_webapi_status_code,- _wpt_delay_seconds,- _wpt_dedicated_worker_js_wrapper_html,- _wpt_any_dedicated_worker_wrapper_html,- _wpt_any_dedicated_worker_wrapper_js,- _wpt_any_window_wrapper_html,- _wpt_window_js_wrapper_html,-)---# WPT fixture tests use loopback servers; inherited shell proxies can intercept-# urllib requests to those servers, so normalize the script process once.-clear_current_proxy_env()---class WptCrossTests(unittest.TestCase):+class WptCrossTests(WptCrossTestCase):def test_cdp_command_failure_marks_page_session_unusable(self) -> None:navigation_response = {"result": {"frameId": "FRAME-1", "loaderId": "LOADER-1"},@@ -193,7 +43,6 @@ class WptCrossTests(unittest.TestCase):self.assertEqual(case_result.status, "error")self.assertTrue((case_result.error or "").startswith(error_prefix))self.assertIn("TimeoutError", case_result.error or "")-def test_navigation_identity_rejects_failed_or_unbound_navigation(self) -> None:with self.assertRaisesRegex(RawCdpError, "ERR_NAME_NOT_RESOLVED"):_navigation_identity(@@ -229,7 +78,6 @@ class WptCrossTests(unittest.TestCase):session_id="SESSION-1",expected_url="http://localhost/case.html",)-def test_navigation_evidence_requires_exact_session_frame_and_loader(self) -> None:identity = _navigation_identity({@@ -279,7 +127,6 @@ class WptCrossTests(unittest.TestCase):self.assertTrue(_has_navigation_evidence([stale, wrong_session, current], identity))-def test_case_path_and_url_normalization_ignore_leading_slash_and_fragment(self) -> None:self.assertEqual(_normalized_case_path("/dom/case.html?variant=1#ignored"),@@ -297,7 +144,6 @@ class WptCrossTests(unittest.TestCase):),"http://localhost:8000/xhr/case.html?load%20fires%20normally",)-def test_run_one_case_rejects_wrong_initial_commit_url(self) -> None:client = SimpleNamespace(send=AsyncMock(return_value=1),@@ -337,7 +183,6 @@ class WptCrossTests(unittest.TestCase):self.assertEqual(result.status, "error")self.assertIn("unexpected URL", result.error or "")self.assertEqual(client.send.await_count, 1)-def test_run_one_case_ignores_stale_payload_until_navigation_is_bound(self) -> None:class FakeClient:def __init__(self) -> None:@@ -440,7 +285,6 @@ class WptCrossTests(unittest.TestCase):sum(command[0] == "Runtime.evaluate" for command in client.commands),2,)-def test_close_page_disposes_context_with_all_auxiliary_targets(self) -> None:class FakeClient:def __init__(self) -> None:@@ -501,7 +345,6 @@ class WptCrossTests(unittest.TestCase):client.commands,)self.assertNotIn("Target.closeTarget", [method for method, _ in client.commands])-def test_unusable_page_session_preserves_case_result_and_relaunches(self) -> None:def handle(name: str) -> SimpleNamespace:return SimpleNamespace(@@ -583,7 +426,6 @@ class WptCrossTests(unittest.TestCase):self.assertIs(result.cases[0], failed_case)self.assertIs(result.cases[1], passed_case)self.assertEqual(driver.shutdown.call_count, 2)-def test_successful_cases_rotate_isolated_page_contexts(self) -> None:handle = SimpleNamespace(process=SimpleNamespace(poll=lambda: None),@@ -659,5093 +501,3 @@ class WptCrossTests(unittest.TestCase):)self.assertEqual(attach.await_count, 2)self.assertEqual(driver.launch.call_count, 1)-- def test_moli_wpt_commands_enable_layout_and_resources(self) -> None:- self.assertEqual(- _moli_command(Path("/bin/moli"), 9222, None),- [- "/bin/moli",- "serve",- "--layout",- "--resource",- "--host",- "127.0.0.1",- "--port",- "9222",- ],- )- fetch = _moli_fetch(- Path("/bin/moli"),- "http://127.0.0.1:8000/case.html",- 30.0,- )- self.assertEqual(- fetch[:5],- [- "/bin/moli",- "fetch",- "--layout",- "--resource",- "http://127.0.0.1:8000/case.html",- ],- )-- def test_harness_timeout_multiplier_uses_wpt_case_metadata(self) -> None:- self.assertEqual(_harness_timeout_multiplier(WptCase("normal.html")), 1.0)- self.assertEqual(- _harness_timeout_multiplier(- WptCase("long.html", timeout_multiplier=LONG_TIMEOUT_MULTIPLIER)- ),- LONG_TIMEOUT_MULTIPLIER,- )-- def test_lightpanda_cli_fetch_keeps_configured_timeout(self) -> None:- command = _lightpanda_fetch(Path("/bin/lightpanda"), "http://127.0.0.1:8000/case.html", 30.0)-- self.assertEqual(command[:4], ["/bin/lightpanda", "fetch", "http://127.0.0.1:8000/case.html", "--dump"])- self.assertEqual(command[command.index("--wait-until") + 1], "done")- self.assertEqual(command[command.index("--wait-ms") + 1], "30000")- self.assertEqual(command[command.index("--http-timeout") + 1], "30000")- self.assertEqual(command[command.index("--terminate-ms") + 1], "30000")- self.assertNotIn("--wait_until", command)- self.assertNotIn("--wait_ms", command)- self.assertNotIn("--http_timeout", command)-- def test_parser_exposes_configurable_parallelism_with_stable_timeout(self) -> None:- parser = _build_parser()- help_text = parser.format_help()-- self.assertEqual(WPT_CROSS_CASE_TIMEOUT_SECONDS, 120.0)- self.assertEqual(WPT_CROSS_PARALLELISM, 50)- self.assertNotIn("--case-timeout", help_text)- self.assertNotIn("--case-timeout-engine", help_text)- self.assertIn("--parallelism", help_text)- self.assertNotIn("--cdp-parallelism", help_text)- self.assertNotIn("--run-order", parser.format_help())- self.assertNotIn("--shuffle-seed", parser.format_help())- required = [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- ]- self.assertEqual(parser.parse_args(required).parallelism, 50)- self.assertEqual(- parser.parse_args([*required, "--parallelism", "7"]).parallelism,- 7,- )-- def test_layout_profiles_are_explicit_and_keep_fixed_parallelism(self) -> None:- parser = _build_parser()- args = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--profile",- "layout",- ]- )-- self.assertEqual(args.profile, "layout")- self.assertEqual(- LAYOUT_PROFILE_DIR_PREFIXES,- (- "css/css-flexbox",- "css/css-grid",- "css/css-sizing",- "css/cssom-view",- ),- )- self.assertEqual(- (LAYOUT_VIEWPORT.width, LAYOUT_VIEWPORT.height),- (800, 600),- )- self.assertEqual(LAYOUT_VIEWPORT.device_scale_factor, 1.0)- self.assertEqual(WPT_CROSS_PARALLELISM, 50)-- def test_all_profile_matrix_deduplicates_default_and_layout_cases(self) -> None:- semantic = WptCase("css/cssom-view/shared.html")- duplicate_layout = WptCase("css/cssom-view/shared.html")- reftest = WptCase(- "css/css-grid/reference.html",- test_type="reftest",- references=(- ReftestReference("css/css-grid/reference-ref.html", "=="),- ),- )-- merged = _deduplicate_cases([semantic, duplicate_layout, reftest])-- self.assertEqual(- [case.case_path for case in merged],- ["css/css-grid/reference.html", "css/cssom-view/shared.html"],- )- self.assertEqual(merged[0].test_type, "reftest")-- def test_manifest_reftest_enumeration_supports_relations_fuzzy_and_filters(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)-- def write_document(rel: str, body: str = "<!doctype html><p>static</p>") -> None:- path = root / rel- path.parent.mkdir(parents=True, exist_ok=True)- path.write_text(body, encoding="utf-8")-- documents = {- "css/css-flexbox/static.html": "<!doctype html><meta name=timeout content=long><p>test</p>",- "css/css-flexbox/ref.html": "<!doctype html><p>reference</p>",- "css/css-flexbox/notref.html": "<!doctype html><p>not reference</p>",- "css/css-flexbox/animation/dynamic.html": "<!doctype html><style>p { animation: pulse 1s }</style>",- "css/css-flexbox/media.html": "<!doctype html><video></video>",- "css/css-flexbox/server.html": "<!doctype html><script src='/handler.py'></script>",- "css/css-flexbox/protocol.h2.html": "<!doctype html><p>h2</p>",- "css/css-flexbox/driver.html": "<!doctype html><p>driver</p>",- }- for rel, body in documents.items():- write_document(rel, body)-- reftests: dict[str, object] = {}-- def add_manifest_item(rel: str, item: list[object]) -> None:- node = reftests- parts = rel.split("/")- for part in parts[:-1]:- child = node.setdefault(part, {})- assert isinstance(child, dict)- node = child- node[parts[-1]] = ["sha", item]-- add_manifest_item(- "css/css-flexbox/static.html",- [- None,- [- ["/css/css-flexbox/ref.html", "=="],- ["/css/css-flexbox/notref.html", "!="],- ],- {- "timeout": "long",- "fuzzy": [- [None, [[0, 1], [0, 2]]],- [- [- "/css/css-flexbox/static.html",- "/css/css-flexbox/notref.html",- "!=",- ],- [[0, 3], [0, 4]],- ],- ],- },- ],- )- for rel in (- "css/css-flexbox/animation/dynamic.html",- "css/css-flexbox/media.html",- "css/css-flexbox/server.html",- "css/css-flexbox/protocol.h2.html",- ):- add_manifest_item(- rel,- [None, [["/css/css-flexbox/ref.html", "=="]], {}],- )- add_manifest_item(- "css/css-flexbox/driver.html",- [- None,- [["/css/css-flexbox/ref.html", "=="]],- {"testdriver": True},- ],- )- (root / "MANIFEST.json").write_text(- json.dumps(- {- "version": 9,- "url_base": "/",- "items": {"reftest": reftests},- }- ),- encoding="utf-8",- )-- cases = enumerate_reftest_cases(- root,- dir_prefixes=("css/css-flexbox",),- )-- self.assertEqual([case.case_path for case in cases], ["css/css-flexbox/static.html"])- case = cases[0]- self.assertEqual(case.test_type, "reftest")- self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)- self.assertEqual(- case.references,- (- ReftestReference(- "css/css-flexbox/ref.html",- "==",- FuzzyTolerance((0, 1), (0, 2)),- ),- ReftestReference(- "css/css-flexbox/notref.html",- "!=",- FuzzyTolerance((0, 3), (0, 4)),- ),- ),- )- self.assertEqual(- explicit_reftest_case(root, "css/css-flexbox/static.html"),- case,- )-- def test_reftest_pixel_comparison_supports_exact_and_fuzzy_bounds(self) -> None:- def captured(image: Image.Image) -> CapturedScreenshot:- stream = io.BytesIO()- image.save(stream, format="PNG")- png = stream.getvalue()- return CapturedScreenshot(- png=png,- sha256=hashlib.sha256(png).hexdigest(),- width=image.width,- height=image.height,- )-- test_image = Image.new("RGB", (4, 4), (0, 0, 0))- reference_image = test_image.copy()- reference_image.putpixel((2, 1), (2, 0, 0))- test_png = captured(test_image)- reference_png = captured(reference_image)-- exact_equal, exact_metrics, exact_diff = compare_reftest_screenshots(- test_png,- reference_png,- None,- )- fuzzy_equal, fuzzy_metrics, fuzzy_diff = compare_reftest_screenshots(- test_png,- reference_png,- FuzzyTolerance((0, 2), (0, 1)),- )- too_strict, _, strict_diff = compare_reftest_screenshots(- test_png,- reference_png,- FuzzyTolerance((0, 1), (0, 1)),- )- self.addCleanup(exact_diff.close)- self.addCleanup(fuzzy_diff.close)- self.addCleanup(strict_diff.close)-- self.assertFalse(exact_equal)- self.assertEqual(exact_metrics["max_difference"], 2)- self.assertEqual(exact_metrics["different_pixels"], 1)- self.assertTrue(fuzzy_equal)- self.assertEqual(- fuzzy_metrics["fuzzy"],- {"max_difference": [0, 2], "total_pixels": [0, 1]},- )- self.assertFalse(too_strict)-- def test_reftest_failure_artifacts_write_test_reference_and_diff_pngs(self) -> None:- def captured(color: tuple[int, int, int]) -> CapturedScreenshot:- image = Image.new("RGB", (3, 2), color)- stream = io.BytesIO()- image.save(stream, format="PNG")- image.close()- png = stream.getvalue()- return CapturedScreenshot(- png=png,- sha256=hashlib.sha256(png).hexdigest(),- width=3,- height=2,- )-- test_png = captured((255, 255, 255))- reference_png = captured((0, 0, 0))- diff_image = Image.new("RGB", (3, 2), (255, 255, 255))- self.addCleanup(diff_image.close)- evidence = _ReftestEvidence(- reference=ReftestReferenceRun(- reference_path="css/example-ref.html",- url="http://example.test/css/example-ref.html",- relation="==",- ),- screenshot=reference_png,- diff_image=diff_image,- )-- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir)- artifacts = _write_reftest_failure_artifacts(- output_dir=output_dir,- engine="moli",- case_path="css/example.html",- test_screenshot=test_png,- evidence=[evidence],- )- written = sorted(- path.name- for path in (output_dir / artifacts["directory"]).glob("*.png")- )- artifact_paths = [- artifacts["test"],- artifacts["references"][0]["reference"],- artifacts["references"][0]["diff"],- ]-- self.assertEqual(- written,- ["diff-01.png", "reference-01.png", "test.png"],- )- self.assertTrue(- all((output_dir / artifact_path).stat().st_size > 0 for artifact_path in artifact_paths)- )-- def test_reftest_match_and_mismatch_relationship_semantics(self) -> None:- self.assertTrue(reftest_relation_passes("==", equal=True))- self.assertFalse(reftest_relation_passes("==", equal=False))- self.assertTrue(reftest_relation_passes("!=", equal=False))- self.assertFalse(reftest_relation_passes("!=", equal=True))- self.assertTrue(- reftest_comparisons_pass(- [- {"relation": "==", "passed": False},- {"relation": "==", "passed": True},- {"relation": "!=", "passed": True},- ]- )- )- self.assertFalse(- reftest_comparisons_pass(- [- {"relation": "==", "passed": True},- {"relation": "!=", "passed": False},- ]- )- )-- def test_fixed_run_schedule_is_deterministic(self) -> None:- cases = [- "html/browsers/a.html",- "html/browsers/b.html",- "content-security-policy/navigation/a.html",- "content-security-policy/navigation/b.html",- "trusted-types/reporting/a.html",- "trusted-types/reporting/b.html",- ]-- scheduled_a, metadata_a = build_run_schedule(- cases,- case_path=lambda case: case,- )- scheduled_b, metadata_b = build_run_schedule(- cases,- case_path=lambda case: case,- )-- self.assertEqual(scheduled_a, scheduled_b)- self.assertEqual(metadata_a, metadata_b)- self.assertCountEqual(scheduled_a, cases)- self.assertNotEqual(scheduled_a, cases)- self.assertEqual(metadata_a["mode"], "fixed-prefix-balanced-shuffle")- self.assertEqual(metadata_a["seed"], FIXED_RUN_SHUFFLE_SEED)- buckets = [case.split("/", 1)[0] for case in scheduled_a]- self.assertTrue(all(left != right for left, right in zip(buckets, buckets[1:])))-- def _run_wpt_cross_with_fake_moli(- self,- *,- output_dir: Path,- known_failures: Path,- case_status: str,- failure_message: str = "expected 555 but got 100",- allow_missing_known_failures: bool = False,- extra_args: list[str] | None = None,- ) -> int:- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.base_url = "http://127.0.0.1:8000"- self.alternate_base_url = "http://127.0.0.1:8001"- self.external_base_url = None- self.external_alternate_base_url = None- self.external_remote_base_url = None- self.external_host = None-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- return None-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- return f"{self.base_url}/{case_path}"-- failures = []- if case_status != "pass":- failures = [{"name": "subtest", "message": failure_message}]- result_dict = {- "engine": "moli",- "binary": "/tmp/moli",- "binary_sha256": "sha",- "binary_version": "0.1.0",- "endpoint": "cli:/tmp/moli",- "ready_ms": None,- "setup_error": None,- "cases": [- {- "case_path": "known.html",- "status": case_status,- "duration_ms": 1.0,- "subtests": {- "total": 1,- "pass": int(case_status == "pass"),- "fail": int(case_status != "pass"),- "timeout": 0,- "notrun": 0,- },- "failures": failures,- "harness_status_name": "OK",- "harness_message": (- "Harness completed with a tracked message"- if case_status != "pass"- else None- ),- "error": None,- }- ],- }- with (- patch(- "moli_benchmark.wpt_cross.__main__.enumerate_cases",- return_value=[WptCase("known.html")],- ),- patch(- "moli_benchmark.wpt_cross.__main__.build_driver",- return_value=SimpleNamespace(cli_fetch_command=["moli"]),- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",- return_value=SimpleNamespace(setup_error=None),- ),- patch(- "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",- return_value=result_dict,- ),- patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),- patch(- "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",- output_dir.parent / "repo-case-lists",- ),- redirect_stdout(StringIO()),- redirect_stderr(StringIO()),- ):- args = [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- str(output_dir),- "--known-failures",- str(known_failures),- ]- if allow_missing_known_failures:- args.append("--allow-missing-known-failures")- if extra_args is not None:- args.extend(extra_args)- return main(args)-- def test_repo_case_lists_are_overwritten_for_primary_engine(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- out_dir = Path(temp_dir)- matrix = [- {- "case_path": "c/timeout.html",- "results": {- "chrome": {"status": "pass"},- "moli": {"status": "timeout"},- },- },- {- "case_path": "b/fail.html",- "results": {- "chrome": {"status": "pass"},- "moli": {"status": "fail"},- },- },- {- "case_path": "a/pass.html",- "results": {- "chrome": {"status": "fail"},- "moli": {"status": "pass"},- },- },- {- "case_path": "d/crash.html",- "results": {- "chrome": {"status": "pass"},- "moli": {"status": "crash"},- },- },- {- "case_path": "f/error.html",- "results": {- "chrome": {"status": "pass"},- "moli": {"status": "error"},- },- },- {- "case_path": "e/stalled.html",- "results": {- "chrome": {"status": "pass"},- "moli": {"status": "harness-stalled"},- },- },- {- "case_path": "g/missing.html",- "results": {- "chrome": {"status": "pass"},- },- },- {- "case_path": "h/unknown.html",- "results": {- "chrome": {"status": "pass"},- "moli": {"status": "unknown"},- },- },- ]-- _write_repo_case_lists(- matrix,- ["chrome", "moli"],- case_list_dir=out_dir,- )-- expected = {- "passed-cases.txt": "a/pass.html\n",- "failed-cases.txt": "b/fail.html\n",- "timeout-cases.txt": "c/timeout.html\n",- "crash-cases.txt": "d/crash.html\n",- "harness-stalled-cases.txt": "e/stalled.html\n",- "error-cases.txt": "f/error.html\n",- "missing-cases.txt": "g/missing.html\n",- "other-cases.txt": "h/unknown.html\n",- }- for file_name, content in expected.items():- self.assertEqual((out_dir / file_name).read_text(encoding="utf-8"), content)-- _write_repo_case_lists(- [- {- "case_path": "only/pass.html",- "results": {"moli": {"status": "pass"}},- }- ],- ["moli"],- case_list_dir=out_dir,- )-- self.assertEqual(- (out_dir / "passed-cases.txt").read_text(encoding="utf-8"),- "only/pass.html\n",- )- for file_name in CASE_LIST_FILES.values():- expected_content = "only/pass.html\n" if file_name == "passed-cases.txt" else ""- self.assertEqual(- (out_dir / file_name).read_text(encoding="utf-8"),- expected_content,- )-- def test_repo_case_lists_only_refresh_for_full_runs(self) -> None:- parser = _build_parser()- full = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- ]- )- explicit_case = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--case",- "custom-elements/Document-createElement.html",- ]- )- dir_prefix = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--dir-prefix",- "shadow-dom",- ]- )- limited = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--limit",- "1",- ]- )- tentative = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--include-tentative",- ]- )- any_js = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--any-js-global",- "window",- ]- )- layout = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--profile",- "layout",- ]- )- all_profiles = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--profile",- "all",- ]- )-- self.assertTrue(_is_full_case_list_run(full))- self.assertTrue(_is_full_case_list_run(all_profiles))- self.assertFalse(_is_full_case_list_run(layout))- self.assertFalse(_is_full_case_list_run(explicit_case))- self.assertFalse(_is_full_case_list_run(dir_prefix))- self.assertFalse(_is_full_case_list_run(limited))- self.assertFalse(_is_full_case_list_run(tentative))- self.assertFalse(_is_full_case_list_run(any_js))-- def test_main_does_not_refresh_repo_case_lists_for_non_full_runs(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)- manifest = root / "known.json"- output_dir = root / "out"- repo_case_lists = root / "repo-case-lists"- repo_case_lists.mkdir()- (repo_case_lists / "passed-cases.txt").write_text(- "existing/full-case.html\n",- encoding="utf-8",- )- manifest.write_text(- json.dumps({"engine": "moli", "rules": []}),- encoding="utf-8",- )-- code = self._run_wpt_cross_with_fake_moli(- output_dir=output_dir,- known_failures=manifest,- case_status="pass",- extra_args=["--limit", "1"],- )-- self.assertEqual(code, 0)- self.assertEqual(- (repo_case_lists / "passed-cases.txt").read_text(encoding="utf-8"),- "existing/full-case.html\n",- )- self.assertFalse((repo_case_lists / "failed-cases.txt").exists())-- def test_case_origin_classification_preserves_secure_context_signal(self) -> None:- worker_case = any_js_case_path_for_global(- "WebCryptoAPI/idlharness.https.any.js",- ANY_JS_DEDICATED_WORKER_GLOBAL,- )- secure_window_case = any_js_case_path_for_global(- "WebCryptoAPI/digest/digest.https.any.js",- ANY_JS_WINDOW_GLOBAL,- )- nonsecure_window_case = any_js_case_path_for_global(- "WebCryptoAPI/historical.any.js",- ANY_JS_WINDOW_GLOBAL,- )-- self.assertTrue(_case_requires_trustworthy_origin(worker_case))- self.assertTrue(_case_requires_trustworthy_origin(secure_window_case))- self.assertFalse(_case_requires_trustworthy_origin(nonsecure_window_case))-- def test_url_for_case_origin_uses_non_loopback_only_for_secure_context_negative_cases(- self,- ) -> None:- class FakeServer:- external_base_url = "http://[2001:db8::1]:9000"-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- base = self.external_base_url if external else "http://127.0.0.1:8000"- return f"{base}/{case_path}"-- secure_case = any_js_case_path_for_global(- "WebCryptoAPI/digest/digest.https.any.js",- ANY_JS_WINDOW_GLOBAL,- )- nonsecure_case = any_js_case_path_for_global(- "WebCryptoAPI/historical.any.js",- ANY_JS_WINDOW_GLOBAL,- )- secure_context_negative_case = "secure-contexts/basic-shared-worker.html"- audio_output_negative_case = "audio-output/secure-context.html"- credential_management_negative_case = (- "credential-management/require_securecontext.html"- )- explicit_http_case = "digital-credentials/non-secure-contexts.http.html"- underscore_insecure_context_case = "web-nfc/nfc_insecure_context.html"- pointer_event_negative_case = "pointerevents/pointerevent_constructor.html"- host_sensitive_case = "webmessaging/with-ports/020.html"-- self.assertTrue(_case_requires_trustworthy_origin(secure_case))- self.assertFalse(_case_requires_trustworthy_origin(nonsecure_case))- self.assertTrue(- _case_requires_non_trustworthy_origin(secure_context_negative_case)- )- self.assertTrue(_case_requires_non_trustworthy_origin(audio_output_negative_case))- self.assertTrue(- _case_requires_non_trustworthy_origin(credential_management_negative_case)- )- self.assertTrue(_case_requires_non_trustworthy_origin(explicit_http_case))- self.assertTrue(- _case_requires_non_trustworthy_origin(underscore_insecure_context_case)- )- self.assertTrue(_case_requires_non_trustworthy_origin(pointer_event_negative_case))- self.assertFalse(_case_requires_non_trustworthy_origin(host_sensitive_case))- self.assertEqual(- _url_for_case_origin(FakeServer(), secure_case, external=False),- f"http://127.0.0.1:8000/{secure_case}",- )- self.assertEqual(- _url_for_case_origin(FakeServer(), nonsecure_case, external=False),- f"http://127.0.0.1:8000/{nonsecure_case}",- )- self.assertEqual(- _url_for_case_origin(- FakeServer(), secure_context_negative_case, external=False- ),- f"http://[2001:db8::1]:9000/{secure_context_negative_case}",- )- self.assertEqual(- _url_for_case_origin(FakeServer(), audio_output_negative_case, external=False),- f"http://[2001:db8::1]:9000/{audio_output_negative_case}",- )- self.assertEqual(- _url_for_case_origin(- FakeServer(), credential_management_negative_case, external=False- ),- f"http://[2001:db8::1]:9000/{credential_management_negative_case}",- )- self.assertEqual(- _url_for_case_origin(FakeServer(), explicit_http_case, external=False),- f"http://[2001:db8::1]:9000/{explicit_http_case}",- )- self.assertEqual(- _url_for_case_origin(- FakeServer(), underscore_insecure_context_case, external=False- ),- f"http://[2001:db8::1]:9000/{underscore_insecure_context_case}",- )- self.assertEqual(- _url_for_case_origin(- FakeServer(), pointer_event_negative_case, external=False- ),- f"http://[2001:db8::1]:9000/{pointer_event_negative_case}",- )- self.assertEqual(- _url_for_case_origin(FakeServer(), host_sensitive_case, external=False),- f"http://127.0.0.1:8000/{host_sensitive_case}",- )- self.assertEqual(- _url_for_case_origin(FakeServer(), secure_case, external=True),- f"http://[2001:db8::1]:9000/{secure_case}",- )-- def test_cli_mode_preserves_case_origin_classification(self) -> None:- calls = []-- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.base_url = "http://127.0.0.1:8000"- self.alternate_base_url = "http://127.0.0.1:8001"- self.external_base_url = "http://[2001:db8::1]:9000"- self.external_alternate_base_url = "http://[2001:db8::1]:9001"- self.external_remote_base_url = "http://[2001:db8::1]:9002"- self.external_host = "2001:db8::1"-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- return None-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- base = self.external_base_url if external else self.base_url- return f"{base}/{case_path}"-- result_dict = {- "engine": "moli",- "binary": "/tmp/moli",- "binary_sha256": "sha",- "binary_version": "0.1.0",- "endpoint": "cli:/tmp/moli",- "ready_ms": None,- "setup_error": None,- "cases": [],- }-- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir) / "out"-- def fake_run_engine_on_cases_cli(**kwargs: object) -> SimpleNamespace:- calls.append(kwargs["cases"])- return SimpleNamespace(setup_error=None)-- with (- patch(- "moli_benchmark.wpt_cross.__main__.enumerate_cases",- return_value=[- WptCase("secure-contexts/basic-shared-worker.html"),- WptCase("digital-credentials/non-secure-contexts.http.html"),- WptCase("WebCryptoAPI/digest/digest.https.html"),- ],- ),- patch(- "moli_benchmark.wpt_cross.__main__.build_driver",- return_value=SimpleNamespace(cli_fetch_command=["moli"]),- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",- side_effect=fake_run_engine_on_cases_cli,- ),- patch(- "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",- return_value=result_dict,- ),- patch(- "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",- Path(temp_dir) / "repo-case-lists",- ),- patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),- redirect_stdout(StringIO()),- redirect_stderr(StringIO()),- ):- code = main(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- str(output_dir),- ]- )-- self.assertEqual(code, 0)- self.assertEqual(- calls,- [- [- (- "secure-contexts/basic-shared-worker.html",- "http://[2001:db8::1]:9000/secure-contexts/basic-shared-worker.html",- WPT_CROSS_CASE_TIMEOUT_SECONDS,- 1.0,- ),- (- "digital-credentials/non-secure-contexts.http.html",- "http://[2001:db8::1]:9000/digital-credentials/non-secure-contexts.http.html",- WPT_CROSS_CASE_TIMEOUT_SECONDS,- 1.0,- ),- (- "WebCryptoAPI/digest/digest.https.html",- "http://127.0.0.1:8000/WebCryptoAPI/digest/digest.https.html",- WPT_CROSS_CASE_TIMEOUT_SECONDS,- 1.0,- )- ]- ],- )-- def test_main_writes_schedule_and_passes_execution_order_to_cli_runner(self) -> None:- captured: dict[str, object] = {}-- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.base_url = "http://127.0.0.1:8000"- self.alternate_base_url = "http://127.0.0.1:8001"- self.external_base_url = None- self.external_alternate_base_url = None- self.external_remote_base_url = None- self.external_host = None-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- return None-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- return f"{self.base_url}/{case_path}"-- cases = [- WptCase("content-security-policy/navigation/a.html"),- WptCase("content-security-policy/navigation/b.html"),- WptCase("html/browsers/a.html"),- WptCase("html/browsers/b.html"),- WptCase("trusted-types/reporting/a.html"),- WptCase("trusted-types/reporting/b.html"),- ]- scheduled_cases, expected_metadata = build_run_schedule(- cases,- case_path=lambda case: case.case_path,- )- result_dict = {- "engine": "moli",- "binary": "/tmp/moli",- "binary_sha256": "sha",- "binary_version": "0.1.0",- "endpoint": "cli:/tmp/moli",- "ready_ms": None,- "setup_error": None,- "cases": [],- }-- def fake_run_engine_on_cases_cli(**kwargs: object) -> SimpleNamespace:- captured["cases"] = kwargs["cases"]- captured["execution_cases"] = kwargs["execution_cases"]- captured["case_timeout_seconds"] = kwargs["case_timeout_seconds"]- captured["parallelism"] = kwargs["parallelism"]- return SimpleNamespace(setup_error=None)-- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir) / "out"- with (- patch(- "moli_benchmark.wpt_cross.__main__.enumerate_cases",- return_value=cases,- ),- patch(- "moli_benchmark.wpt_cross.__main__.build_driver",- return_value=SimpleNamespace(cli_fetch_command=["moli"]),- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",- side_effect=fake_run_engine_on_cases_cli,- ),- patch(- "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",- return_value=result_dict,- ),- patch(- "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",- Path(temp_dir) / "repo-case-lists",- ),- patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),- redirect_stdout(StringIO()),- redirect_stderr(StringIO()),- ):- code = main(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- str(output_dir),- ]- )- self.assertEqual(code, 0)- self.assertEqual(- (output_dir / "cases.txt").read_text(encoding="utf-8").splitlines(),- [case.case_path for case in cases],- )- self.assertEqual(- (output_dir / "schedule.txt").read_text(encoding="utf-8").splitlines(),- [case.case_path for case in scheduled_cases],- )- schedule_json = json.loads(- (output_dir / "schedule.json").read_text(encoding="utf-8")- )- self.assertEqual(schedule_json, expected_metadata)- self.assertEqual(- schedule_json["mode"],- "fixed-prefix-balanced-shuffle",- )- self.assertEqual(- [case[0] for case in captured["cases"]],- [case.case_path for case in cases],- )- self.assertEqual(- [case[0] for case in captured["execution_cases"]],- [case.case_path for case in scheduled_cases],- )- self.assertEqual(captured["case_timeout_seconds"], WPT_CROSS_CASE_TIMEOUT_SECONDS)- self.assertEqual(captured["parallelism"], WPT_CROSS_PARALLELISM)-- def test_main_uses_requested_parallelism_for_cdp_runner(self) -> None:- calls: list[dict[str, object]] = []-- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.base_url = "http://127.0.0.1:8000"- self.alternate_base_url = "http://127.0.0.1:8001"- self.external_base_url = None- self.external_alternate_base_url = None- self.external_remote_base_url = None- self.external_host = None-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- return None-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- return f"{self.base_url}/{case_path}"-- def fake_run_engine_on_cases(**kwargs: object) -> EngineRunResult:- cases_arg = kwargs["cases"]- assert isinstance(cases_arg, list)- calls.append(- {- "cases": cases_arg,- "case_timeout_seconds": kwargs["case_timeout_seconds"],- }- )- case_path, url, _timeout = cases_arg[0]- return EngineRunResult(- engine="chrome",- binary="/tmp/chrome",- binary_sha256="sha",- binary_version="version",- endpoint="cdp://127.0.0.1:1",- ready_ms=1.0,- cases=[CaseResult(case_path, url, "pass", 1.0)],- )-- result_dict = {- "engine": "chrome",- "binary": "/tmp/chrome",- "binary_sha256": "sha",- "binary_version": "version",- "endpoint": "cdp://127.0.0.1:1",- "ready_ms": 1.0,- "setup_error": None,- "cases": [],- }-- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir) / "out"- cases = [WptCase(f"case-{index}.html") for index in range(3)]- with (- patch(- "moli_benchmark.wpt_cross.__main__.enumerate_cases",- return_value=cases,- ),- patch(- "moli_benchmark.wpt_cross.__main__.build_driver",- return_value=SimpleNamespace(cli_fetch_command=None),- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases",- side_effect=fake_run_engine_on_cases,- ),- patch(- "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",- return_value=result_dict,- ),- patch(- "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",- Path(temp_dir) / "repo-case-lists",- ),- patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),- redirect_stdout(StringIO()),- redirect_stderr(StringIO()),- ):- code = main(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "chrome",- "--output-dir",- str(output_dir),- "--mode",- "cdp",- "--parallelism",- "2",- ]- )-- self.assertEqual(code, 0)- self.assertEqual(len(calls), 2)- self.assertTrue(all(call["case_timeout_seconds"] == WPT_CROSS_CASE_TIMEOUT_SECONDS for call in calls))- self.assertEqual(- sorted(case[0] for call in calls for case in call["cases"]),- [case.case_path for case in cases],- )-- def test_layout_testharness_profile_forces_cdp_and_fixed_viewport(self) -> None:- captured: dict[str, object] = {}-- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.base_url = "http://127.0.0.1:8000"- self.alternate_base_url = "http://127.0.0.1:8001"- self.external_base_url = None- self.external_alternate_base_url = None- self.external_remote_base_url = None- self.external_host = None-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- return None-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- return f"{self.base_url}/{case_path}"-- def fake_cdp_run(**kwargs: object) -> EngineRunResult:- captured.update(kwargs)- cases_arg = kwargs["cases"]- assert isinstance(cases_arg, list)- case_path, url, _timeout = cases_arg[0]- return EngineRunResult(- engine="moli",- binary="/tmp/moli",- binary_sha256="sha",- binary_version="version",- endpoint="cdp://127.0.0.1:1",- ready_ms=1.0,- cases=[CaseResult(case_path, url, "pass", 1.0)],- )-- case = WptCase("css/css-flexbox/layout.html")- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir) / "out"- with (- patch(- "moli_benchmark.wpt_cross.__main__.enumerate_cases",- return_value=[case],- ) as enumerate_mock,- patch(- "moli_benchmark.wpt_cross.__main__.build_driver",- return_value=SimpleNamespace(cli_fetch_command=lambda *_: []),- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases",- side_effect=fake_cdp_run,- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",- side_effect=AssertionError("layout profile must not use CLI mode"),- ),- patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),- redirect_stdout(StringIO()),- redirect_stderr(StringIO()),- ):- code = main(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- str(output_dir),- "--profile",- "layout-testharness",- ]- )-- summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))-- self.assertEqual(code, 0)- self.assertEqual(captured["viewport"], LAYOUT_VIEWPORT)- self.assertIsNone(captured["artifact_output_dir"])- self.assertTrue(enumerate_mock.call_args.kwargs["layout_static_only"])- self.assertEqual(summary["profile"], "layout-testharness")- self.assertEqual(summary["viewport"]["width"], 800)-- def test_main_uses_wpt_harness_multipliers_independent_of_outer_deadline(self) -> None:- calls = []-- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.base_url = "http://127.0.0.1:8000"- self.alternate_base_url = "http://127.0.0.1:8001"- self.external_base_url = None- self.external_alternate_base_url = None- self.external_remote_base_url = None- self.external_host = None-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- calls.append((multipliers, default_multiplier))-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- return f"{self.base_url}/{case_path}"-- result_dict = {- "engine": "moli",- "binary": "/tmp/moli",- "binary_sha256": "sha",- "binary_version": "0.1.0",- "endpoint": "cli:/tmp/moli",- "ready_ms": None,- "setup_error": None,- "cases": [],- }- cases = [- WptCase("normal.html", timeout_multiplier=1.0),- WptCase("long.html", timeout_multiplier=LONG_TIMEOUT_MULTIPLIER),- WptCase(- "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html",- timeout_multiplier=1.0,- ),- ]-- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir) / "out"- with (- patch(- "moli_benchmark.wpt_cross.__main__.enumerate_cases",- return_value=cases,- ),- patch(- "moli_benchmark.wpt_cross.__main__.build_driver",- return_value=SimpleNamespace(cli_fetch_command=["moli"]),- ),- patch(- "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",- return_value=SimpleNamespace(setup_error=None),- ),- patch(- "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",- return_value=result_dict,- ),- patch(- "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",- Path(temp_dir) / "repo-case-lists",- ),- patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),- redirect_stdout(StringIO()),- redirect_stderr(StringIO()),- ):- code = main(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- str(output_dir),- ]- )-- self.assertEqual(code, 0)- self.assertEqual(- calls,- [- (- {- "normal.html": 1.0,- "long.html": LONG_TIMEOUT_MULTIPLIER,- "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html": 1.0,- },- 1.0,- )- ],- )-- def test_normal_dynamic_import_uses_normal_wpt_timeout(self) -> None:- case = WptCase(- "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html"- )-- self.assertEqual(_harness_timeout_multiplier(case), 1.0)-- def test_parser_rejects_removed_timeout_flags_and_legacy_parallelism(self) -> None:- parser = _build_parser()- base = [- "--wpt-root",- "/tmp/wpt",- "--engine",- "chrome",- "--output-dir",- "/tmp/out",- ]- for removed_flag in (- "--case-timeout",- "--case-timeout-engine",- "--cdp-parallelism",- ):- with self.subTest(removed_flag=removed_flag), self.assertRaises(SystemExit):- parser.parse_args([*base, removed_flag, "1"])-- for invalid_parallelism in ("0", "-1", "not-an-integer"):- with self.subTest(invalid_parallelism=invalid_parallelism), self.assertRaises(- SystemExit- ):- parser.parse_args(- [*base, "--parallelism", invalid_parallelism]- )-- def test_main_matrix_preserves_harness_message(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)- manifest = root / "known.json"- output_dir = root / "out"- manifest.write_text(- json.dumps(- {- "engine": "moli",- "rules": [- {- "case_path": "known.html",- "expected_status": "fail",- "message_contains": "tracked message",- }- ],- }- ),- encoding="utf-8",- )-- code = self._run_wpt_cross_with_fake_moli(- output_dir=output_dir,- known_failures=manifest,- case_status="fail",- failure_message="expected 555 but got 100",- )-- self.assertEqual(code, 0)- matrix = json.loads((output_dir / "matrix.json").read_text(encoding="utf-8"))- result = matrix[0]["results"]["moli"]- self.assertEqual(- result["harness_message"],- "Harness completed with a tracked message",- )- audit = json.loads(- (output_dir / "known-failure-audit-moli.json").read_text(- encoding="utf-8"- )- )- self.assertEqual(- audit["known_failures"][0]["harness_message"],- "Harness completed with a tracked message",- )-- def test_build_partial_preserves_diagnostic_fields(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir)- (output_dir / "engine-moli.json").write_text(- json.dumps(- {- "cases": [- {- "case_path": "known.html",- "status": "fail",- "duration_ms": 1.0,- "subtests": {- "total": 0,- "pass": 0,- "fail": 0,- "timeout": 0,- "notrun": 0,- },- "harness_status_name": "ERROR",- "harness_message": "Unhandled rejection: cycle",- "error": "testharness completed without reporting any subtests",- "test_type": "reftest",- "failures": [{"name": "== known-ref.html"}],- "failure_names": ["== known-ref.html"],- "reftest_comparisons": [- {- "reference_path": "known-ref.html",- "relation": "==",- "passed": False,- "max_difference": 255,- "different_pixels": 10,- }- ],- "artifacts": {- "test": "artifacts/moli/known/test.png",- "references": [- {- "reference": "artifacts/moli/known/reference-01.png",- "diff": "artifacts/moli/known/diff-01.png",- }- ],- },- }- ]- }- ),- encoding="utf-8",- )-- with redirect_stdout(StringIO()):- code = build_partial_main([str(output_dir), "--engine", "moli"])-- self.assertEqual(code, 0)- matrix = json.loads(- (output_dir / "matrix.partial.moli.json").read_text(- encoding="utf-8"- )- )- self.assertEqual(- matrix[0]["results"]["moli"]["harness_message"],- "Unhandled rejection: cycle",- )- self.assertEqual(matrix[0]["test_type"], "reftest")- result = matrix[0]["results"]["moli"]- self.assertEqual(result["failure_names"], ["== known-ref.html"])- self.assertEqual(result["reftest_comparisons"][0]["relation"], "==")- self.assertEqual(- result["artifacts"]["test"],- "artifacts/moli/known/test.png",- )-- def test_parser_accepts_known_failure_audit_options(self) -> None:- parser = _build_parser()- args = parser.parse_args(- [- "--wpt-root",- "/tmp/wpt",- "--engine",- "moli",- "--output-dir",- "/tmp/out",- "--known-failures",- "/tmp/known.json",- "--known-failures-engine",- "moli",- "--allow-missing-known-failures",- ]- )-- self.assertEqual(args.known_failures, Path("/tmp/known.json"))- self.assertEqual(args.known_failures_engine, "moli")- self.assertTrue(args.allow_missing_known_failures)-- def test_main_writes_known_failure_audit_when_manifest_matches(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)- docs = root / "docs"- wpt = root / "wpt"- docs.mkdir()- wpt.mkdir()- (docs / "wasm-current.md").write_text("# wasm\n", encoding="utf-8")- (wpt / "known.html").write_text("<!doctype html>", encoding="utf-8")- manifest = root / "known.json"- output_dir = root / "out"- manifest.write_text(- json.dumps(- {- "engine": "moli",- "categories": {- "wasm": {- "tracking_doc": "docs/wasm-current.md",- "scope": "tracked wasm failure",- "evidence": [- {- "kind": "doc",- "path": "docs/wasm-current.md",- "note": "local fixture evidence",- },- {- "kind": "wpt",- "path": "wpt/known.html",- "note": "known failure source fixture",- }- ],- }- },- "rules": [- {- "case_path": "known.html",- "category": "wasm",- "expected_status": "fail",- "message_contains": "expected 555",- "reason": "tracked wasm failure",- }- ],- }- ),- encoding="utf-8",- )-- code = self._run_wpt_cross_with_fake_moli(- output_dir=output_dir,- known_failures=manifest,- case_status="fail",- )-- self.assertEqual(code, 0)- audit = json.loads(- (output_dir / "known-failure-audit-moli.json").read_text(- encoding="utf-8"- )- )- self.assertTrue(audit["ok"])- self.assertEqual(audit["counts"]["known_failures"], 1)- self.assertEqual(audit["categories"]["wasm"]["scope"], "tracked wasm failure")- summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))- self.assertTrue(summary["known_failure_audits"]["moli"]["ok"])- self.assertEqual(- summary["known_failure_audits"]["moli"]["categories"]["wasm"][- "tracking_doc"- ],- "docs/wasm-current.md",- )- self.assertEqual(- summary["known_failure_audits"]["moli"]["category_counts"][- "known_failures"- ],- {"wasm": 1},- )-- def test_main_can_skip_missing_known_failures_for_focused_runs(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)- manifest = root / "known.json"- output_dir = root / "out"- manifest.write_text(- json.dumps(- {- "engine": "moli",- "rules": [- {- "case_path": "known.html",- "expected_status": "fail",- "message_contains": "expected 555",- },- {- "case_path": "not-run.html",- "expected_status": "fail",- },- ],- }- ),- encoding="utf-8",- )-- code = self._run_wpt_cross_with_fake_moli(- output_dir=output_dir,- known_failures=manifest,- case_status="fail",- allow_missing_known_failures=True,- )-- self.assertEqual(code, 0)- audit = json.loads(- (output_dir / "known-failure-audit-moli.json").read_text(- encoding="utf-8"- )- )- self.assertTrue(audit["ok"])- self.assertEqual(audit["counts"]["known_failures"], 1)- self.assertEqual(audit["counts"]["missing_expected_failures"], 0)- self.assertEqual(audit["counts"]["skipped_known_failures"], 1)- summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))- self.assertEqual(- summary["known_failure_audits"]["moli"]["counts"][- "skipped_known_failures"- ],- 1,- )-- def test_main_returns_nonzero_when_known_failure_audit_finds_unexpected_failure(- self,- ) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)- manifest = root / "known.json"- output_dir = root / "out"- manifest.write_text(- json.dumps({"engine": "moli", "rules": []}), encoding="utf-8"- )-- code = self._run_wpt_cross_with_fake_moli(- output_dir=output_dir,- known_failures=manifest,- case_status="fail",- )-- self.assertEqual(code, 5)- audit = json.loads(- (output_dir / "known-failure-audit-moli.json").read_text(- encoding="utf-8"- )- )- self.assertFalse(audit["ok"])- self.assertEqual(audit["counts"]["unexpected_failures"], 1)-- def test_main_returns_nonzero_when_known_failure_is_resolved(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- root = Path(temp_dir)- manifest = root / "known.json"- output_dir = root / "out"- manifest.write_text(- json.dumps(- {- "engine": "moli",- "rules": [- {- "case_path": "known.html",- "expected_status": "fail",- }- ],- }- ),- encoding="utf-8",- )-- code = self._run_wpt_cross_with_fake_moli(- output_dir=output_dir,- known_failures=manifest,- case_status="pass",- )-- self.assertEqual(code, 5)- audit = json.loads(- (output_dir / "known-failure-audit-moli.json").read_text(- encoding="utf-8"- )- )- self.assertFalse(audit["ok"])- self.assertEqual(audit["counts"]["resolved_known_failures"], 1)-- def test_enumerate_cases_includes_htm_before_applying_limit(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "dom"- case_dir.mkdir()- harness = '<script src="/resources/testharness.js"></script>'- (case_dir / "a.htm").write_text(- '<meta name="timeout" content="long">'- '<meta name="variant" content="?mode=z">'- '<meta name="variant" content="?mode=a">'- + harness,- encoding="utf-8",- )- (case_dir / "z.html").write_text(harness, encoding="utf-8")- expected = [- WptCase("dom/a.htm?mode=a", LONG_TIMEOUT_MULTIPLIER),- WptCase("dom/a.htm?mode=z", LONG_TIMEOUT_MULTIPLIER),- WptCase("dom/z.html"),- ]-- for prefixes in (None, ("dom",)):- with self.subTest(dir_prefixes=prefixes):- self.assertEqual(- enumerate_cases(wpt_root, dir_prefixes=prefixes), expected- )- self.assertEqual(- enumerate_cases(wpt_root, dir_prefixes=prefixes, limit=1),- expected[:1],- )-- def test_enumerate_cases_expands_wpt_meta_variants(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "dom" / "ranges"- case_dir.mkdir(parents=True)- (case_dir / "variant.html").write_text(- """<!doctype html>-<meta name="variant" content="?mode=open">-<meta name="variant" content="?mode=closed#frag">-<meta name="variant" content="?command=heading¶m=h1">-<meta name="variant" content="?mode=encoded&flag=1">-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (case_dir / "plain.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (case_dir / "long.html").write_text(- """<!doctype html>-<meta name="timeout" content="long">-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("dom/ranges",))-- self.assertEqual(- [case.case_path for case in cases],- [- "dom/ranges/long.html",- "dom/ranges/plain.html",- "dom/ranges/variant.html?command=heading¶m=h1",- "dom/ranges/variant.html?mode=closed#frag",- "dom/ranges/variant.html?mode=encoded&flag=1",- "dom/ranges/variant.html?mode=open",- ],- )- timeouts = {case.case_path: case.timeout_multiplier for case in cases}- self.assertEqual(timeouts["dom/ranges/long.html"], LONG_TIMEOUT_MULTIPLIER)- self.assertEqual(timeouts["dom/ranges/plain.html"], 1.0)- self.assertEqual(timeouts["dom/ranges/variant.html?mode=open"], 1.0)-- def test_enumerate_cases_skips_wptserve_python_resources(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "html" / "semantics"- common_dir = wpt_root / "common" / "security-features" / "resources"- case_dir.mkdir(parents=True)- common_dir.mkdir(parents=True)- (case_dir / "direct.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>fetch("/resources/inspect.py?cmd=get");</script>-""",- encoding="utf-8",- )- (case_dir / "external.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="helper.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (case_dir / "helper.js").write_text(- 'fetch("/resources/inspect-header.py?cmd=get");',- encoding="utf-8",- )- (case_dir / "kept.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (case_dir / "generated-comment.html").write_text(- """<!doctype html>-<!-- DO NOT EDIT! This test has been generated by /html/canvas/tools/gentest.py. -->-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (case_dir / "absolute-helper.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/common/security-features/resources/common.sub.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (common_dir / "common.sub.js").write_text(- 'fetch("/common/security-features/subresource/xhr.py");',- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("html/semantics",))-- self.assertEqual(- [case.case_path for case in cases],- [- "html/semantics/generated-comment.html",- "html/semantics/kept.html",- ],- )-- def test_default_enumeration_uses_non_layout_blacklist_without_encoding(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- dom_dir = wpt_root / "dom" / "nodes"- encoding_dir = wpt_root / "encoding" / "legacy-mb-japanese"- canvas_dir = wpt_root / "html" / "canvas" / "element"- layout_dir = wpt_root / "css" / "css-grid"- container_query_dir = (- wpt_root / "css" / "css-conditional" / "container-queries"- )- css_conditional_dir = wpt_root / "css" / "css-conditional"- css_animation_dir = wpt_root / "css" / "css-animations" / "parsing"- css_animation_timeline_dir = wpt_root / "css" / "css-animations"- css_transition_dir = wpt_root / "css" / "css-transitions"- css_transition_parsing_dir = (- wpt_root / "css" / "css-transitions" / "parsing"- )- css_paint_dir = wpt_root / "css" / "css-paint-api"- css_scrollbar_dir = wpt_root / "css" / "css-scrollbars"- css_viewport_dir = wpt_root / "css" / "css-viewport" / "zoom"- css_viewport_parsing_dir = (- wpt_root / "css" / "css-viewport" / "zoom" / "parsing"- )- typed_om_dir = wpt_root / "css" / "css-typed-om"- render_blocking_dir = wpt_root / "html" / "dom" / "render-blocking"- service_worker_dir = wpt_root / "service-workers" / "service-worker"- service_worker_named_dir = wpt_root / "webusb"- non_service_worker_https_dir = wpt_root / "WebCryptoAPI"- media_dir = (- wpt_root / "html" / "semantics" / "embedded-content" / "media-elements"- )- iframe_dir = (- wpt_root- / "html"- / "semantics"- / "embedded-content"- / "the-iframe-element"- )- dom_dir.mkdir(parents=True)- encoding_dir.mkdir(parents=True)- canvas_dir.mkdir(parents=True)- layout_dir.mkdir(parents=True)- container_query_dir.mkdir(parents=True)- css_conditional_dir.mkdir(parents=True, exist_ok=True)- css_animation_dir.mkdir(parents=True)- css_transition_dir.mkdir(parents=True)- css_transition_parsing_dir.mkdir(parents=True)- css_paint_dir.mkdir(parents=True)- css_scrollbar_dir.mkdir(parents=True)- css_viewport_dir.mkdir(parents=True)- css_viewport_parsing_dir.mkdir(parents=True)- typed_om_dir.mkdir(parents=True)- render_blocking_dir.mkdir(parents=True)- service_worker_dir.mkdir(parents=True)- service_worker_named_dir.mkdir(parents=True)- non_service_worker_https_dir.mkdir(parents=True)- media_dir.mkdir(parents=True)- iframe_dir.mkdir(parents=True)- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-"""- (dom_dir / "kept.html").write_text(case_html, encoding="utf-8")- (encoding_dir / "kept.html").write_text(case_html, encoding="utf-8")- (canvas_dir / "filtered.html").write_text(case_html, encoding="utf-8")- (layout_dir / "filtered.html").write_text(case_html, encoding="utf-8")- (container_query_dir / "at-container-parsing.html").write_text(- case_html, encoding="utf-8"- )- (container_query_dir / "size-feature-evaluation.html").write_text(- case_html, encoding="utf-8"- )- (css_conditional_dir / "at-supports-named-feature-001.html").write_text(- case_html, encoding="utf-8"- )- (css_animation_dir / "kept.html").write_text(case_html, encoding="utf-8")- (css_animation_timeline_dir / "animate-with-color-mix.html").write_text(- case_html, encoding="utf-8"- )- (css_transition_dir / "events-001.html").write_text(- case_html, encoding="utf-8"- )- (css_transition_parsing_dir / "transition-computed.html").write_text(- case_html, encoding="utf-8"- )- (css_paint_dir / "filtered.html").write_text(case_html, encoding="utf-8")- (css_scrollbar_dir / "scrollbar-width-001.html").write_text(- case_html, encoding="utf-8"- )- (css_scrollbar_dir / "scrollbar-width-parsing.html").write_text(- case_html, encoding="utf-8"- )- (css_viewport_dir / "widget.html").write_text(case_html, encoding="utf-8")- (css_viewport_parsing_dir / "zoom-computed.html").write_text(- case_html, encoding="utf-8"- )- (typed_om_dir / "kept.html").write_text(case_html, encoding="utf-8")- (render_blocking_dir / "filtered.html").write_text(- case_html, encoding="utf-8"- )- (service_worker_dir / "registration.https.html").write_text(- case_html, encoding="utf-8"- )- (service_worker_named_dir / "usb.serviceworker.https.html").write_text(- case_html, encoding="utf-8"- )- (- non_service_worker_https_dir- / "crypto-subtle-secure-context-available.https.sub.html"- ).write_text(case_html, encoding="utf-8")- (media_dir / "filtered.html").write_text(case_html, encoding="utf-8")- (iframe_dir / "kept.html").write_text(case_html, encoding="utf-8")-- cases = enumerate_cases(wpt_root)- focused_transition_cases = enumerate_cases(- wpt_root,- dir_prefixes=("css/css-transitions",),- )-- self.assertNotIn("encoding", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertIn("html/canvas", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertIn("css/css-grid", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertNotIn(- "css/css-conditional/container-queries", DEFAULT_EXCLUDE_DIR_PREFIXES- )- self.assertIn("css/css-paint-api", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertNotIn("css/css-animations", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertNotIn("css/css-transitions", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertIn("html/dom/render-blocking", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertNotIn("service-workers", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertIn(- "html/semantics/embedded-content/media-elements",- DEFAULT_EXCLUDE_DIR_PREFIXES,- )- self.assertNotIn("css/css-typed-om", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertNotIn(- "html/semantics/embedded-content/the-iframe-element",- DEFAULT_EXCLUDE_DIR_PREFIXES,- )- self.assertEqual(- [case.case_path for case in cases],- [- "css/css-animations/parsing/kept.html",- "css/css-conditional/container-queries/at-container-parsing.html",- "css/css-scrollbars/scrollbar-width-parsing.html",- "css/css-transitions/parsing/transition-computed.html",- "css/css-typed-om/kept.html",- "css/css-viewport/zoom/parsing/zoom-computed.html",- "dom/nodes/kept.html",- "encoding/legacy-mb-japanese/kept.html",- "html/semantics/embedded-content/the-iframe-element/kept.html",- "service-workers/service-worker/registration.https.html",- "webusb/usb.serviceworker.https.html",- ],- )- self.assertEqual(- [case.case_path for case in focused_transition_cases],- [- "css/css-transitions/events-001.html",- "css/css-transitions/parsing/transition-computed.html",- ],- )-- def test_default_enumeration_excludes_jpegxl_codec_fidelity(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- jpegxl_dir = wpt_root / "jpegxl"- jpegxl_dir.mkdir(parents=True)- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-"""- (jpegxl_dir / "decode.html").write_text(case_html, encoding="utf-8")-- default_cases = enumerate_cases(wpt_root)- focused_cases = enumerate_cases(wpt_root, dir_prefixes=("jpegxl",))-- self.assertIn("jpegxl", DEFAULT_EXCLUDE_DIR_PREFIXES)- self.assertEqual(default_cases, [])- self.assertEqual(- [case.case_path for case in focused_cases],- ["jpegxl/decode.html"],- )-- def test_enumerate_cases_skips_manual_and_support_resource_pages(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "FileAPI"- resource_dir = wpt_root / "appmanifest" / "resources"- support_dir = wpt_root / "workers" / "support"- case_dir.mkdir(parents=True)- resource_dir.mkdir(parents=True)- support_dir.mkdir(parents=True)- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-"""- for suffix in (".html", ".htm"):- (case_dir / f"kept{suffix}").write_text(case_html, encoding="utf-8")- (case_dir / f"upload-manual{suffix}").write_text(- case_html, encoding="utf-8"- )- (resource_dir / f"helper{suffix}").write_text(- case_html, encoding="utf-8"- )- (support_dir / f"helper{suffix}").write_text(- case_html, encoding="utf-8"- )-- cases = enumerate_cases(wpt_root)-- self.assertEqual(- [case.case_path for case in cases],- ["FileAPI/kept.htm", "FileAPI/kept.html"],- )-- def test_explicit_dir_prefix_bypasses_default_rendering_blacklist(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- canvas_dir = wpt_root / "html" / "canvas"- canvas_dir.mkdir(parents=True)- (canvas_dir / "case.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("html/canvas",))-- self.assertEqual([case.case_path for case in cases], ["html/canvas/case.html"])-- def test_preload_dir_prefix_allows_modulepreload_helper_without_stash_usage(- self,- ) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- preload_dir = wpt_root / "preload"- resources_dir = preload_dir / "resources"- resources_dir.mkdir(parents=True)- harness_dir = wpt_root / "resources"- harness_dir.mkdir(parents=True)- (harness_dir / "testharness.js").write_text("", encoding="utf-8")- (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")- (resources_dir / "preload_helper.js").write_text(- "function unused() { return '/preload/resources/stash-put.py'; }",- encoding="utf-8",- )- modulepreload_case = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script src="/preload/resources/preload_helper.js"></script>-<link rel=modulepreload href="resources/dummy.js?pipe=trickle(d5)">-<script>promise_test(async () => {}, "ok");</script>-"""- (preload_dir / "avoid-delaying-onload-link-modulepreload.html").write_text(- modulepreload_case,- encoding="utf-8",- )- (- preload_dir / "avoid-delaying-onload-link-modulepreload-exec.html"- ).write_text(- modulepreload_case,- encoding="utf-8",- )- (preload_dir / "avoid-delaying-onload-link-preload.html").write_text(- modulepreload_case,- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("preload",))-- self.assertEqual(- [case.case_path for case in cases],- [- "preload/avoid-delaying-onload-link-modulepreload-exec.html",- "preload/avoid-delaying-onload-link-modulepreload.html",- ],- )-- def test_preload_dir_prefix_excludes_sri_non_goal_cases(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- preload_dir = wpt_root / "preload"- preload_dir.mkdir(parents=True)- harness_dir = wpt_root / "resources"- harness_dir.mkdir(parents=True)- (harness_dir / "testharness.js").write_text("", encoding="utf-8")- (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>promise_test(async () => {}, "ok");</script>-"""- (preload_dir / "modulepreload-json.html").write_text(- case_html,- encoding="utf-8",- )- (preload_dir / "modulepreload-sri.html").write_text(- case_html,- encoding="utf-8",- )- (preload_dir / "modulepreload-sri-importmap.html").write_text(- case_html,- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("preload",))-- self.assertEqual(- [case.case_path for case in cases],- ["preload/modulepreload-json.html"],- )-- def test_enumerate_cases_excludes_malformed_select_fragment_non_goal(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- parsing_dir = wpt_root / "html" / "syntax" / "parsing"- parsing_dir.mkdir(parents=True)- harness_dir = wpt_root / "resources"- harness_dir.mkdir(parents=True)- (harness_dir / "testharness.js").write_text("", encoding="utf-8")- (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>test(() => {}, "ok");</script>-"""- (parsing_dir / "html5lib_innerHTML_webkit02.html").write_text(- case_html,- encoding="utf-8",- )- (parsing_dir / "html5lib_scripted_webkit01.html").write_text(- case_html,- encoding="utf-8",- )-- default_cases = enumerate_cases(wpt_root)- focused_cases = enumerate_cases(- wpt_root,- dir_prefixes=("html/syntax/parsing",),- )-- expected = ["html/syntax/parsing/html5lib_scripted_webkit01.html"]- self.assertEqual([case.case_path for case in default_cases], expected)- self.assertEqual([case.case_path for case in focused_cases], expected)-- def test_enumerate_cases_excludes_formdata_non_goal_cases(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- formdata_dir = wpt_root / "xhr" / "formdata"- formdata_dir.mkdir(parents=True)- harness_dir = wpt_root / "resources"- harness_dir.mkdir(parents=True)- (harness_dir / "testharness.js").write_text("", encoding="utf-8")- (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>test(() => {}, "ok");</script>-"""- (formdata_dir / "constructor-formelement.html").write_text(- case_html,- encoding="utf-8",- )- (formdata_dir / "constructor-submitter-coordinate.html").write_text(- case_html,- encoding="utf-8",- )- (formdata_dir / "constructor-submitter.html").write_text(- case_html,- encoding="utf-8",- )-- default_cases = enumerate_cases(wpt_root)- focused_cases = enumerate_cases(- wpt_root,- dir_prefixes=("xhr/formdata",),- )- diagnostic_case = explicit_case(- wpt_root,- "xhr/formdata/constructor-formelement.html",- )- coordinate_diagnostic_case = explicit_case(- wpt_root,- "xhr/formdata/constructor-submitter-coordinate.html",- )-- expected = ["xhr/formdata/constructor-submitter.html"]- self.assertEqual([case.case_path for case in default_cases], expected)- self.assertEqual([case.case_path for case in focused_cases], expected)- self.assertEqual(- diagnostic_case.case_path,- "xhr/formdata/constructor-formelement.html",- )- self.assertEqual(- coordinate_diagnostic_case.case_path,- "xhr/formdata/constructor-submitter-coordinate.html",- )-- def test_enumerate_cases_allows_supported_xhr_delay_handler(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- xhr_dir = wpt_root / "xhr"- xhr_dir.mkdir(parents=True)- harness_dir = wpt_root / "resources"- harness_dir.mkdir(parents=True)- (harness_dir / "testharness.js").write_text("", encoding="utf-8")- (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")- (xhr_dir / "delay-supported.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>fetch("resources/delay.py?ms=1"); test(() => {}, "ok");</script>-""",- encoding="utf-8",- )- (xhr_dir / "delay-and-unsupported.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>fetch("resources/delay.py"); fetch("resources/other.py");</script>-""",- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("xhr",))-- self.assertEqual(- [case.case_path for case in cases],- ["xhr/delay-supported.html"],- )-- def test_enumerate_cases_allows_supported_delayed_module_handler(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- harness_dir = wpt_root / "resources"- harness_dir.mkdir(parents=True)- (harness_dir / "testharness.js").write_text("", encoding="utf-8")- (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")- module_dir = (- wpt_root- / "html"- / "semantics"- / "scripting-1"- / "the-script-element"- / "module"- )- (module_dir / "resources").mkdir(parents=True)- (module_dir / "delay-supported.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script type="module" src="./delay-supported.js"></script>-""",- encoding="utf-8",- )- (module_dir / "delay-supported.js").write_text(- 'import "./resources/delayed-modulescript.py?ms=1";',- encoding="utf-8",- )- (module_dir / "resources" / "delayed-modulescript.py").write_text(- "# modeled by wpt-cross",- encoding="utf-8",- )-- cases = enumerate_cases(- wpt_root,- dir_prefixes=(- "html/semantics/scripting-1/the-script-element/module",- ),- )-- self.assertEqual(- [case.case_path for case in cases],- [- "html/semantics/scripting-1/the-script-element/module/"- "delay-supported.html"- ],- )-- def test_enumerate_cases_wraps_window_js_for_explicit_dir_prefix(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- api_dir = wpt_root / "WebCryptoAPI"- api_dir.mkdir(parents=True)- (api_dir / "algorithm-discards-context.https.window.js").write_text(- """// META: title=window case-// META: timeout=long-promise_test(async () => {}, "ok");-""",- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root, dir_prefixes=("WebCryptoAPI",))-- self.assertEqual(- [case.case_path for case in cases],- [- "WebCryptoAPI/algorithm-discards-context.https.window.js?moli-wpt-script=window",- ],- )- self.assertEqual(cases[0].timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)-- def test_explicit_dir_prefix_includes_simple_https_sub_html(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- api_dir = wpt_root / "WebCryptoAPI" / "secure_context"- api_dir.mkdir(parents=True)- (api_dir / "crypto-subtle-secure-context-available.https.sub.html").write_text(- """<!doctype html>-<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>test(() => assert_equals("{{host}}", "{{host}}"), "ok");</script>-""",- encoding="utf-8",- )-- default_cases = enumerate_cases(wpt_root)- selected_cases = enumerate_cases(wpt_root, dir_prefixes=("WebCryptoAPI",))-- self.assertEqual(default_cases, [])- self.assertEqual(- [case.case_path for case in selected_cases],- ["WebCryptoAPI/secure_context/crypto-subtle-secure-context-available.https.sub.html"],- )-- def test_dir_prefix_can_opt_into_tentative_cases(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "wasm" / "webapi" / "esm-integration"- case_dir.mkdir(parents=True)- case_html = """<!doctype html>-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-"""- (case_dir / "stable.html").write_text(case_html, encoding="utf-8")- (case_dir / "feature.tentative.html").write_text(- case_html,- encoding="utf-8",- )-- default_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/webapi/esm-integration",),- )- tentative_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/webapi/esm-integration",),- include_tentative=True,- )-- self.assertEqual(- [case.case_path for case in default_cases],- ["wasm/webapi/esm-integration/stable.html"],- )- self.assertEqual(- [case.case_path for case in tentative_cases],- [- "wasm/webapi/esm-integration/feature.tentative.html",- "wasm/webapi/esm-integration/stable.html",- ],- )-- def test_dir_prefix_can_opt_into_any_js_globals(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "wasm" / "jsapi" / "js-string"- resource_dir = case_dir / "resources"- case_dir.mkdir(parents=True)- resource_dir.mkdir(parents=True)- (case_dir / "basic.any.js").write_text(- """// META: timeout=long-promise_test(async () => {}, "ok");-""",- encoding="utf-8",- )- (case_dir / "feature.tentative.any.js").write_text(- """promise_test(async () => {}, "ok");""",- encoding="utf-8",- )- (case_dir / "testdriver.any.js").write_text(- """// META: script=/resources/testdriver.js-promise_test(async () => {}, "requires testdriver");-""",- encoding="utf-8",- )- (resource_dir / "helper.any.js").write_text(- """promise_test(async () => {}, "not a top-level case");""",- encoding="utf-8",- )-- default_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/jsapi/js-string",),- )- window_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/jsapi/js-string",),- any_js_global=ANY_JS_WINDOW_GLOBAL,- )- both_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/jsapi/js-string",),- include_tentative=True,- any_js_global="both",- )-- self.assertEqual(default_cases, [])- self.assertEqual(- [(case.case_path, case.timeout_multiplier) for case in window_cases],- [- (- "wasm/jsapi/js-string/basic.any.js?moli-wpt-any=window",- LONG_TIMEOUT_MULTIPLIER,- )- ],- )- self.assertEqual(- [case.case_path for case in both_cases],- [- "wasm/jsapi/js-string/basic.any.js?moli-wpt-any=dedicatedworker",- "wasm/jsapi/js-string/basic.any.js?moli-wpt-any=window",- "wasm/jsapi/js-string/feature.tentative.any.js?moli-wpt-any=dedicatedworker",- "wasm/jsapi/js-string/feature.tentative.any.js?moli-wpt-any=window",- ],- )-- def test_default_enumeration_includes_streams_script_cases(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- streams_dir = wpt_root / "streams" / "readable-streams"- other_dir = wpt_root / "wasm" / "jsapi"- streams_dir.mkdir(parents=True)- other_dir.mkdir(parents=True)- (streams_dir / "general.any.js").write_text(- """// META: global=window,worker-promise_test(async () => {}, "general");-""",- encoding="utf-8",- )- (streams_dir / "window-only.any.js").write_text(- """// META: global=window-test(() => {}, "window only");-""",- encoding="utf-8",- )- (streams_dir / "task.window.js").write_text(- """// META: timeout=long-test(() => {}, "window script");-""",- encoding="utf-8",- )- (streams_dir / "task.worker.js").write_text(- """test(() => {}, "worker script");-done();-""",- encoding="utf-8",- )- (streams_dir / "feature.tentative.any.js").write_text(- """test(() => {}, "tentative");""",- encoding="utf-8",- )- (other_dir / "excluded.any.js").write_text(- """test(() => {}, "not in the broad baseline");""",- encoding="utf-8",- )-- cases = enumerate_cases(wpt_root)-- self.assertEqual(- [(case.case_path, case.timeout_multiplier) for case in cases],- [- (- "streams/readable-streams/general.any.js?moli-wpt-any=dedicatedworker",- 1.0,- ),- (- "streams/readable-streams/general.any.js?moli-wpt-any=window",- 1.0,- ),- (- "streams/readable-streams/task.window.js?moli-wpt-script=window",- LONG_TIMEOUT_MULTIPLIER,- ),- (- "streams/readable-streams/task.worker.js?moli-wpt-script=dedicatedworker",- 1.0,- ),- (- "streams/readable-streams/window-only.any.js?moli-wpt-any=window",- 1.0,- ),- ],- )-- def test_default_enumeration_includes_compression_script_cases(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- sources = {- "compression/compression-stream.any.js": "// META: global=window,worker\n",- "compression/idlharness.https.any.js": "// META: global=window,dedicatedworker\n",- "compression/compression-with-detach.window.js": "",- "compression/decompression-with-detach.window.js": "",- "compression/feature.tentative.any.js": "",- "compression/testdriver.any.js": "// META: script=/resources/testdriver.js\n",- "compression/resources/helper.any.js": "",- "compression-other/excluded.any.js": "",- "compression-other/excluded.window.js": "",- }- for name, meta in sources.items():- path = wpt_root / name- path.parent.mkdir(parents=True, exist_ok=True)- path.write_text(meta + 'test(() => {}, "case");', encoding="utf-8")-- cases = enumerate_cases(wpt_root)- focused_cases = enumerate_cases(- wpt_root, dir_prefixes=("compression",), any_js_global="both"- )- window_cases = enumerate_cases(wpt_root, any_js_global="window")-- expected = [- "compression/compression-stream.any.js?moli-wpt-any=dedicatedworker",- "compression/compression-stream.any.js?moli-wpt-any=window",- "compression/compression-with-detach.window.js?moli-wpt-script=window",- "compression/decompression-with-detach.window.js?moli-wpt-script=window",- "compression/idlharness.https.any.js?moli-wpt-any=dedicatedworker",- "compression/idlharness.https.any.js?moli-wpt-any=window",- ]- self.assertEqual([case.case_path for case in cases], expected)- self.assertEqual([case.case_path for case in focused_cases], expected)- self.assertEqual(- [case.case_path for case in window_cases if case.case_path.startswith("compression/")],- [path for path in expected if "dedicatedworker" not in path],- )-- def test_dir_prefix_expands_any_js_variants_and_respects_globals(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "WebCryptoAPI"- case_dir.mkdir(parents=True)- (case_dir / "multi.any.js").write_text(- """// META: global=window,worker-// META: timeout=long-// META: variant=?mode=1-// META: variant=?mode=2-promise_test(async () => {}, "multi");-""",- encoding="utf-8",- )- (case_dir / "window-only.any.js").write_text(- """// META: global=window-promise_test(async () => {}, "window");-""",- encoding="utf-8",- )- (case_dir / "worker-only.any.js").write_text(- """// META: global=dedicatedworker-promise_test(async () => {}, "worker");-""",- encoding="utf-8",- )-- cases = enumerate_cases(- wpt_root,- dir_prefixes=("WebCryptoAPI",),- any_js_global="both",- )-- self.assertEqual(- [(case.case_path, case.timeout_multiplier) for case in cases],- [- (- "WebCryptoAPI/multi.any.js?mode=1&moli-wpt-any=dedicatedworker",- LONG_TIMEOUT_MULTIPLIER,- ),- (- "WebCryptoAPI/multi.any.js?mode=1&moli-wpt-any=window",- LONG_TIMEOUT_MULTIPLIER,- ),- (- "WebCryptoAPI/multi.any.js?mode=2&moli-wpt-any=dedicatedworker",- LONG_TIMEOUT_MULTIPLIER,- ),- (- "WebCryptoAPI/multi.any.js?mode=2&moli-wpt-any=window",- LONG_TIMEOUT_MULTIPLIER,- ),- (- "WebCryptoAPI/window-only.any.js?moli-wpt-any=window",- 1.0,- ),- (- "WebCryptoAPI/worker-only.any.js?moli-wpt-any=dedicatedworker",- 1.0,- ),- ],- )-- def test_dir_prefix_includes_window_and_worker_js_wrappers(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "wasm" / "serialization"- case_dir.mkdir(parents=True)- (case_dir / "transfer.window.js").write_text(- """// META: timeout=long-test(() => {}, "window");-""",- encoding="utf-8",- )- (case_dir / "memory.worker.js").write_text(- """importScripts("/resources/testharness.js");-test(() => {}, "worker");-done();-""",- encoding="utf-8",- )- (case_dir / "feature.tentative.window.js").write_text(- """test(() => {}, "tentative");""",- encoding="utf-8",- )-- default_cases = enumerate_cases(wpt_root)- focused_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/serialization",),- )- tentative_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm/serialization",),- include_tentative=True,- )-- self.assertEqual(default_cases, [])- self.assertEqual(- [(case.case_path, case.timeout_multiplier) for case in focused_cases],- [- (- "wasm/serialization/memory.worker.js?moli-wpt-script=dedicatedworker",- 1.0,- ),- (- "wasm/serialization/transfer.window.js?moli-wpt-script=window",- LONG_TIMEOUT_MULTIPLIER,- ),- ],- )- self.assertEqual(- [case.case_path for case in tentative_cases],- [- "wasm/serialization/feature.tentative.window.js?moli-wpt-script=window",- "wasm/serialization/memory.worker.js?moli-wpt-script=dedicatedworker",- "wasm/serialization/transfer.window.js?moli-wpt-script=window",- ],- )-- def test_dir_prefix_includes_sub_https_and_supported_wasm_status_handler(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- serialization_dir = wpt_root / "wasm" / "serialization" / "module"- serialization_dir.mkdir(parents=True)- webapi_dir = wpt_root / "wasm" / "webapi"- webapi_dir.mkdir(parents=True)- esm_dir = webapi_dir / "esm-integration"- esm_dir.mkdir()- (serialization_dir / "window-domain-success.sub.html").write_text(- """<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>test(() => {}, "sub");</script>-""",- encoding="utf-8",- )- (esm_dir / "worklet-import-source-phase.tentative.https.html").write_text(- """<script src="/resources/testharness.js"></script>-<script src="/resources/testharnessreport.js"></script>-<script>test(() => {}, "https tentative");</script>-""",- encoding="utf-8",- )- (webapi_dir / "status.any.js").write_text(- """// META: global=window,worker-promise_test(async t => {- await promise_rejects_js(t, TypeError, WebAssembly.compileStreaming(fetch("status.py?status=404")));-}, "status");-""",- encoding="utf-8",- )- (webapi_dir / "badstatus.any.js").write_text(- """// META: global=window,worker-promise_test(async t => {- await promise_rejects_js(t, TypeError, WebAssembly.compileStreaming(fetch("badstatus.py?status=404")));-}, "badstatus");-""",- encoding="utf-8",- )- (webapi_dir / "origin.sub.any.js").write_text(- """// META: global=window,worker-promise_test(async t => {- await promise_rejects_js(t, TypeError, WebAssembly.compileStreaming(fetch("/fetch/api/resources/redirect.py?redirect_status=301&location=/wasm/incrementer.wasm")));-}, "origin");-""",- encoding="utf-8",- )-- broad_cases = enumerate_cases(wpt_root)- focused_cases = enumerate_cases(- wpt_root,- dir_prefixes=("wasm",),- include_tentative=True,- any_js_global="both",- )-- self.assertEqual(broad_cases, [])- self.assertEqual(- [case.case_path for case in focused_cases],- [- "wasm/serialization/module/window-domain-success.sub.html",- "wasm/webapi/esm-integration/worklet-import-source-phase.tentative.https.html",- "wasm/webapi/origin.sub.any.js?moli-wpt-any=dedicatedworker",- "wasm/webapi/origin.sub.any.js?moli-wpt-any=window",- "wasm/webapi/status.any.js?moli-wpt-any=dedicatedworker",- "wasm/webapi/status.any.js?moli-wpt-any=window",- ],- )-- def test_explicit_case_bypasses_curated_filename_filters(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "shadow-dom"- case_dir.mkdir(parents=True)- (case_dir / "feature.tentative.html").write_text(- """<!doctype html>-<meta name="timeout" content="long">-<script src="/resources/testharness.js"></script>-<script>test(() => {}, "ok");</script>-""",- encoding="utf-8",- )-- case = explicit_case(wpt_root, "shadow-dom/feature.tentative.html?variant=1")-- self.assertEqual(case.case_path, "shadow-dom/feature.tentative.html?variant=1")- self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)-- def test_explicit_case_wraps_any_js(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "WebCryptoAPI"- case_dir.mkdir(parents=True)- (case_dir / "historical.any.js").write_text(- """// META: global=window,dedicatedworker-// META: timeout=long-test(() => {}, "ok");-""",- encoding="utf-8",- )-- case = explicit_case(wpt_root, "WebCryptoAPI/historical.any.js?mode=1")-- self.assertEqual(- case.case_path,- "WebCryptoAPI/historical.any.js?mode=1&moli-wpt-any=window",- )- self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)-- def test_explicit_window_and_worker_js_cases_use_wrappers(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "wasm"- case_dir.mkdir(parents=True)- (case_dir / "transfer.window.js").write_text(- """// META: timeout=long-test(() => {}, "ok");-""",- encoding="utf-8",- )- (case_dir / "memory.worker.js").write_text(- """test(() => {}, "ok");""",- encoding="utf-8",- )-- window_case = explicit_case(wpt_root, "wasm/transfer.window.js?variant=1")- worker_case = explicit_case(wpt_root, "wasm/memory.worker.js")-- self.assertEqual(- window_case.case_path,- "wasm/transfer.window.js?variant=1&moli-wpt-script=window",- )- self.assertEqual(window_case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)- self.assertEqual(- worker_case.case_path,- "wasm/memory.worker.js?moli-wpt-script=dedicatedworker",- )-- def test_explicit_case_accepts_generated_any_html_path(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "WebCryptoAPI"- case_dir.mkdir(parents=True)- (case_dir / "historical.any.js").write_text(- """// META: timeout=long-test(() => {}, "ok");-""",- encoding="utf-8",- )-- case = explicit_case(- wpt_root,- f"WebCryptoAPI/historical.any.html?{ANY_JS_WINDOW_QUERY}",- )-- self.assertEqual(- case.case_path,- f"WebCryptoAPI/historical.any.html?{ANY_JS_WINDOW_QUERY}",- )- self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)-- def test_explicit_case_accepts_generated_window_html_path(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- wpt_root = Path(temp_dir)- case_dir = wpt_root / "WebCryptoAPI"- case_dir.mkdir(parents=True)- (case_dir / "algorithm-discards-context.https.window.js").write_text(- """// META: timeout=long-test(() => {}, "ok");-""",- encoding="utf-8",- )-- case = explicit_case(- wpt_root,- f"WebCryptoAPI/algorithm-discards-context.https.window.html?{WINDOW_JS_WINDOW_QUERY}",- )-- self.assertEqual(- case.case_path,- f"WebCryptoAPI/algorithm-discards-context.https.window.html?{WINDOW_JS_WINDOW_QUERY}",- )- self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)-- def test_any_js_window_case_path_does_not_duplicate_window_query(self) -> None:- self.assertEqual(- any_js_window_case_path(- f"WebCryptoAPI/historical.any.js?{ANY_JS_WINDOW_QUERY}#frag"- ),- f"WebCryptoAPI/historical.any.html?{ANY_JS_WINDOW_QUERY}#frag",- )-- def test_window_js_window_case_path_does_not_duplicate_window_query(self) -> None:- self.assertEqual(- window_js_window_case_path(- f"WebCryptoAPI/example.window.js?{WINDOW_JS_WINDOW_QUERY}#frag"- ),- f"WebCryptoAPI/example.window.html?{WINDOW_JS_WINDOW_QUERY}#frag",- )-- def test_any_js_metadata_matches_wpt_header_rules(self) -> None:- meta = parse_any_js_meta(- """//META: variant=?first-// META: script=helper.js--// META: variant=?ignored-"""- )-- self.assertEqual(meta.variants, ["?first"])- self.assertEqual(meta.scripts, ["helper.js"])-- def test_report_bridge_keys_results_by_initial_search_without_hash(self) -> None:- self.assertIn(b"var initialCasePath =", BENCH_REPORT_BRIDGE)- self.assertIn(b"case_path: initialCasePath", BENCH_REPORT_BRIDGE)- self.assertIn(b"location.pathname + location.search", BENCH_REPORT_BRIDGE)- self.assertNotIn(b"location.pathname + location.search + location.hash", BENCH_REPORT_BRIDGE)-- def test_report_bridge_does_not_post_incremental_payloads(self) -> None:- self.assertIn(b"if (source === 'incremental') {", BENCH_REPORT_BRIDGE)- incremental_start = BENCH_REPORT_BRIDGE.index(b"if (source === 'incremental') {")- final_payload_start = BENCH_REPORT_BRIDGE.index(b"var snapshot = {")- incremental_block = BENCH_REPORT_BRIDGE[incremental_start:final_payload_start]-- self.assertIn(b"partial_count: accumulator.tests.length", incremental_block)- self.assertIn(b"tests: []", incremental_block)- self.assertIn(b"return;", incremental_block)- self.assertNotIn(b"accumulator.tests.slice()", incremental_block)-- def test_report_bridge_posts_only_bounded_final_payloads_synchronously(self) -> None:- self.assertIn(b"if (body.length <= 60000)", BENCH_REPORT_BRIDGE)- self.assertIn(b"xhr.open('POST', '/__bench__/result', false)", BENCH_REPORT_BRIDGE)- self.assertLess(- BENCH_REPORT_BRIDGE.index(b"xhr.open('POST', '/__bench__/result', false)"),- BENCH_REPORT_BRIDGE.rindex(b"window.__bench_wpt__ = snapshot;"),- )-- def test_report_bridge_writes_dom_payload_before_posting(self) -> None:- self.assertIn(b"__bench_wpt_payload", BENCH_REPORT_BRIDGE)- self.assertLess(- BENCH_REPORT_BRIDGE.index(b"node.textContent = body;"),- BENCH_REPORT_BRIDGE.index(b"xhr.open('POST', '/__bench__/result', false)"),- )-- def test_report_bridge_disables_in_page_testharness_output(self) -> None:- self.assertIn(b"output: false", BENCH_REPORT_BRIDGE)-- def test_report_bridge_applies_configured_timeout_multiplier(self) -> None:- bridge = _bench_report_bridge(7.5)-- self.assertIn(b"timeout_multiplier: 7.5", bridge)-- def test_report_bridge_config_is_injected_without_changing_case_url(self) -> None:- body = (- b'<!doctype html><script src="/resources/testharness.js"></script>'- b'<script src="/resources/testharnessreport.js"></script>'- b"<script>test(() => location.search, 'query visible to case')</script>"- )-- injected = _inject_bench_report_bridge_config(body, 3.0)-- self.assertIn(- (- b'src="/resources/testharnessreport.js?'- + BENCH_TIMEOUT_MULTIPLIER_QUERY.encode("ascii")- + b'=3"'- ),- injected,- )- self.assertIn(b"location.search", injected)-- def test_report_bridge_config_keeps_default_multiplier_body_unchanged(self) -> None:- body = (- b'<!doctype html><script src="/resources/testharness.js"></script>'- b'<script src="/resources/testharnessreport.js"></script>'- )-- self.assertEqual(_inject_bench_report_bridge_config(body, 1.0), body)-- def test_harness_case_key_ignores_url_fragment(self) -> None:- self.assertEqual(- _normalize_harness_case_key("/dom/ranges/feature.html?mode=open#frag"),- "dom/ranges/feature.html?mode=open",- )-- def test_report_bridge_does_not_publish_empty_done_fallback(self) -> None:- self.assertIn(b"function canPublishDoneFallback()", BENCH_REPORT_BRIDGE)- self.assertIn(- b"window.__bench_wpt__ === undefined && canPublishDoneFallback()",- BENCH_REPORT_BRIDGE,- )- self.assertIn(- b"canPublishDoneFallback()) {\n publish('done-hook-late', null);",- BENCH_REPORT_BRIDGE,- )-- def test_case_result_dict_records_payload_source(self) -> None:- row = case_result_to_dict(- CaseResult(- case_path="example.html",- url="http://example.test/example.html",- status="pass",- duration_ms=1.0,- payload_source="completion-callback",- )- )-- self.assertEqual(row["payload_source"], "completion-callback")-- def test_case_result_dict_records_reftest_comparisons_and_artifacts(self) -> None:- row = case_result_to_dict(- CaseResult(- case_path="css/example.html",- url="http://example.test/css/example.html",- status="fail",- duration_ms=1.0,- test_type="reftest",- reftest_comparisons=[- {- "reference_path": "css/example-ref.html",- "relation": "==",- "passed": False,- "max_difference": 255,- "different_pixels": 10,- }- ],- artifacts={- "test": "artifacts/moli/example/test.png",- "references": [- {- "reference": "artifacts/moli/example/reference-01.png",- "diff": "artifacts/moli/example/diff-01.png",- }- ],- },- )- )-- self.assertEqual(row["test_type"], "reftest")- self.assertEqual(row["reftest_comparisons"][0]["relation"], "==")- self.assertEqual(row["artifacts"]["test"], "artifacts/moli/example/test.png")-- def test_cli_runner_extracts_payload_from_stdout_html(self) -> None:- payload = {- "case_path": "/case.html",- "harness": {"status": 0, "message": None},- "tests": [{"name": "ok", "status": 0}],- "source": "completion-callback",- }- stdout = (- "<!doctype html><html><pre id=\"__bench_wpt_payload\" hidden>"- + json.dumps(payload)- + "</pre></html>"- ).encode()-- self.assertEqual(_payload_from_stdout_html(stdout), payload)-- def test_cli_runner_prefers_final_stdout_payload_without_callback_wait(self) -> None:- class UnexpectedResultsAccess:- def wait_for_final(self, key: str, timeout: float) -> None:- raise AssertionError("final stdout payload should skip callback wait")-- def get(self, key: str) -> None:- raise AssertionError("final stdout payload should skip callback lookup")-- payload = {- "case_path": "/case.html",- "harness": {"status": 0, "message": None},- "tests": [{"name": "ok", "status": 0}],- "source": "completion-callback",- }- stdout = (- '<!doctype html><pre id="__bench_wpt_payload" hidden>'- + json.dumps(payload)- + "</pre>"- ).encode()-- result = _classify_cli_case_result(- case_path="case.html",- url="http://example.test/case.html",- bridge_key="/case.html",- fixture_server=SimpleNamespace(results=UnexpectedResultsAccess()),- subprocess_result=_CliSubprocessResult(- duration_ms=10.0,- proc_error=None,- proc_returncode=0,- proc_stderr="",- proc_stdout=stdout,- wait_script_timeout=False,- ),- payload_grace_seconds=2.0,- successful_process_payload_grace_seconds=8.0,- )-- self.assertEqual(result.status, "pass")- self.assertEqual(result.payload_source, "completion-callback")-- def test_case_result_dict_records_failure_details(self) -> None:- row = case_result_to_dict(- CaseResult(- case_path="example.html",- url="http://example.test/example.html",- status="fail",- duration_ms=1.0,- failures=[- {- "name": "subtest name",- "status": 1,- "status_name": "FAIL",- "message": "expected true got false",- }- ],- )- )-- self.assertEqual(- row["failures"],- [- {- "name": "subtest name",- "status": 1,- "status_name": "FAIL",- "message": "expected true got false",- }- ],- )-- def test_classify_payload_records_limited_failure_details(self) -> None:- result = classify_payload(- payload={- "source": "completion-callback",- "harness": {"status": 0},- "tests": [- {"name": "ok", "status": 0},- {- "name": "bad",- "status": 1,- "message": "x" * 700,- },- {"name": "notrun", "status": 3},- ],- },- case_path="example.html",- url="http://example.test/example.html",- duration_ms=1.0,- bridge_installed=True,- )-- self.assertEqual(result.status, "fail")- self.assertEqual(result.subtests_total, 3)- self.assertEqual([failure["name"] for failure in result.failures], ["bad", "notrun"])- self.assertEqual(result.failure_names, ["bad", "notrun"])- self.assertEqual(result.failures[0]["status_name"], "FAIL")- self.assertEqual(len(result.failures[0]["message"]), 500)- self.assertTrue(result.failures[0]["message_truncated"])-- def test_recorded_failure_drift_reports_hidden_subtest_differences(self) -> None:- drift = _recorded_failure_drift(- [- {- "case_path": "webcrypto/shared-fail.html",- "results": {- "moli": {- "status": "fail",- "failures": [- {"name": "shared", "message": "lm detail"},- {"name": "lm-only", "message": "bad"},- ],- },- "chrome": {- "status": "fail",- "failures": [- {"name": "shared", "message": "chrome detail"},- {"name": "chrome-only", "message": "bad"},- ],- },- },- }- ],- ["moli", "chrome"],- )-- self.assertEqual(drift["comparison_count"], 1)- self.assertEqual(drift["primary_only_comparison_count"], 1)- self.assertEqual(drift["peer_only_comparison_count"], 1)- self.assertEqual(drift["message_diff_comparison_count"], 1)- row = drift["comparisons"][0]- self.assertEqual(row["primary_only_examples"], ["lm-only"])- self.assertEqual(row["peer_only_examples"], ["chrome-only"])- self.assertEqual(row["message_diff_examples"], ["shared"])-- def test_recorded_failure_drift_uses_full_failure_names_when_available(self) -> None:- drift = _recorded_failure_drift(- [- {- "case_path": "webcrypto/large-shared-fail.html",- "results": {- "moli": {- "status": "fail",- "failures": [{"name": "recorded", "message": "same"}],- "failure_names": ["recorded", "late-moli-only"],- },- "chrome": {- "status": "fail",- "failures": [{"name": "recorded", "message": "same"}],- "failure_names": ["recorded"],- },- },- }- ],- ["moli", "chrome"],- )-- self.assertEqual(drift["comparison_count"], 1)- self.assertEqual(drift["primary_only_comparison_count"], 1)- self.assertEqual(drift["comparisons"][0]["primary_only_examples"], ["late-moli-only"])-- def test_classify_payload_does_not_pass_incremental_only_result(self) -> None:- result = classify_payload(- payload={- "source": "incremental",- "harness": {"status": None},- "tests": [{"name": "observed pass", "status": 0}],- },- case_path="example.html",- url="http://example.test/example.html",- duration_ms=1.0,- bridge_installed=True,- )-- self.assertEqual(result.status, "harness-stalled")- self.assertEqual(result.subtests_total, 1)- self.assertEqual(result.subtests_pass, 1)- self.assertEqual(result.payload_source, "incremental")- self.assertIn("incremental", result.error or "")-- def test_classify_payload_does_not_pass_empty_final_result(self) -> None:- result = classify_payload(- payload={- "source": "completion-callback",- "harness": {"status": 0},- "tests": [],- },- case_path="example.html",- url="http://example.test/example.html",- duration_ms=1.0,- bridge_installed=True,- )-- self.assertEqual(result.status, "fail")- self.assertEqual(result.subtests_total, 0)- self.assertIn("without reporting any subtests", result.error or "")-- def test_classify_payload_includes_empty_harness_error_message(self) -> None:- result = classify_payload(- payload={- "source": "completion-callback",- "harness": {- "status": 1,- "message": "Unhandled rejection: cyclic wasm dependency",- },- "tests": [],- },- case_path="example.html",- url="http://example.test/example.html",- duration_ms=1.0,- bridge_installed=True,- )-- self.assertEqual(result.status, "fail")- self.assertEqual(result.harness_status, 1)- self.assertEqual(result.subtests_total, 0)- self.assertIn("without reporting any subtests", result.error or "")- self.assertIn("cyclic wasm dependency", result.error or "")-- def test_classify_payload_keeps_observed_incremental_failure(self) -> None:- result = classify_payload(- payload={- "source": "incremental",- "harness": {"status": None},- "tests": [{"name": "observed fail", "status": 1}],- },- case_path="example.html",- url="http://example.test/example.html",- duration_ms=1.0,- bridge_installed=True,- )-- self.assertEqual(result.status, "fail")- self.assertEqual(result.subtests_fail, 1)- self.assertEqual(result.payload_source, "incremental")-- def test_results_store_wait_for_final_does_not_return_incremental(self) -> None:- store = ResultsStore()- store.put("example.html", {"source": "incremental"})-- self.assertIsNone(store.wait_for_final("example.html", timeout=0))- self.assertEqual(store.get("example.html"), {"source": "incremental"})-- def test_testdriver_vendor_bridge_provides_action_sequence(self) -> None:- self.assertIn(b"action_sequence", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"pointerMove", BENCH_TESTDRIVER_VENDOR_BRIDGE)-- def test_testdriver_vendor_bridge_preserves_key_modifiers(self) -> None:- self.assertIn(b"altKey: !!modifiers.Alt", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"ctrlKey: !!modifiers.Control", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"shiftKey: !!modifiers.Shift", BENCH_TESTDRIVER_VENDOR_BRIDGE)-- def test_testdriver_vendor_bridge_focuses_user_activation_target(self) -> None:- self.assertIn(b"focusForUserActivation", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"document.activeElement === before", BENCH_TESTDRIVER_VENDOR_BRIDGE)-- def test_testdriver_vendor_bridge_falls_back_to_recorded_hit_test_targets(self) -> None:- self.assertIn(b"nativeGetClientRects.apply", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"nativeElementsFromPoint.apply", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"return target ? [target] : []", BENCH_TESTDRIVER_VENDOR_BRIDGE)-- def test_testdriver_vendor_bridge_accepts_storage_access_permission_setup(self) -> None:- self.assertIn(- b"params.descriptor.name === 'storage-access'",- BENCH_TESTDRIVER_VENDOR_BRIDGE,- )- self.assertIn(- b"set_permission() is not implemented by the Moli WPT bridge",- BENCH_TESTDRIVER_VENDOR_BRIDGE,- )-- def test_testdriver_vendor_bridge_provides_computed_label(self) -> None:- self.assertIn(b"get_computed_label", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"resolveReferenceTarget", BENCH_TESTDRIVER_VENDOR_BRIDGE)- self.assertIn(b"data-expectedlabel", BENCH_TESTDRIVER_VENDOR_BRIDGE)-- def test_fixture_server_parses_wpt_trickle_pipe_delay(self) -> None:- self.assertEqual(_pipe_trickle_delay_seconds("pipe=trickle(d1)&cachebust=1"), 1.0)- self.assertEqual(_pipe_trickle_delay_seconds("pipe=header(X,Y)|trickle(d2.5)"), 2.5)- self.assertEqual(- _pipe_trickle_delay_seconds("pipe=trickle(d3)&pipe=trickle(d1)"),- 3.0,- )- self.assertEqual(_pipe_trickle_delay_seconds("pipe=trickle(d999)"), 10.0)- self.assertEqual(_pipe_trickle_delay_seconds("notpipe=trickle(d1)"), 0.0)-- def test_fixture_server_parses_delay_handler_duration(self) -> None:- self.assertEqual(_wpt_delay_seconds("ms=3000"), 3.0)- self.assertEqual(_wpt_delay_seconds("ms=2.5"), 0.0025)- self.assertEqual(_wpt_delay_seconds("ms=250&ms=750"), 0.25)- self.assertEqual(_wpt_delay_seconds(""), 0.5)- self.assertIsNone(_wpt_delay_seconds("ms=invalid"))- self.assertIsNone(_wpt_delay_seconds("ms=-1"))- self.assertIsNone(_wpt_delay_seconds("ms=nan"))-- def test_fixture_server_models_xhr_delay_py_methods(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- with WptFixtureServer(root_path) as server:- with patch(- "moli_benchmark.wpt_cross.server.time.sleep"- ) as sleep_mock:- url = f"{server.base_url}/xhr/resources/delay.py?ms=250"- responses = []- for method in ("GET", "HEAD", "POST", "OPTIONS", "YO"):- request = Request(- url,- data=b"upload" if method in {"POST", "YO"} else None,- method=method,- )- with urlopen(request, timeout=2) as response:- responses.append(- (- method,- response.status,- response.read(),- response.headers["Content-Type"],- response.headers["Access-Control-Allow-Origin"],- response.headers["Access-Control-Allow-Methods"],- )- )-- self.assertEqual(- [call.args for call in sleep_mock.call_args_list],- [(0.25,)] * 5,- )- self.assertEqual(- responses,- [- ("GET", 200, b"TEST_DELAY", "text/plain", "*", "YO"),- ("HEAD", 200, b"", "text/plain", "*", "YO"),- ("POST", 200, b"TEST_DELAY", "text/plain", "*", "YO"),- ("OPTIONS", 200, b"TEST_DELAY", "text/plain", "*", "YO"),- ("YO", 200, b"TEST_DELAY", "text/plain", "*", "YO"),- ],- )-- def test_fixture_server_models_delayed_module_script_handler(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- with WptFixtureServer(root_path) as server:- with patch(- "moli_benchmark.wpt_cross.server.time.sleep"- ) as sleep_mock:- url = (- f"{server.base_url}/html/semantics/scripting-1/"- "the-script-element/module/resources/"- "delayed-modulescript.py?ms=250"- )- responses = []- for method in ("GET", "HEAD"):- request = Request(url, method=method)- with urlopen(request, timeout=2) as response:- responses.append(- (- method,- response.status,- response.read(),- response.headers["Content-Type"],- )- )-- self.assertEqual(- [call.args for call in sleep_mock.call_args_list],- [(0.25,), (0.25,)],- )- self.assertEqual(- responses,- [- (- "GET",- 200,- b"export let delayedLoaded = true;",- "text/javascript",- ),- ("HEAD", 200, b"", "text/javascript"),- ],- )-- def test_fixture_server_models_common_redirect_opt_in_handler(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- with WptFixtureServer(root_path) as server:- connection = HTTPConnection("127.0.0.1", server.port, timeout=2)- connection.request(- "GET",- "/common/redirect-opt-in.py?status=307&location=%2Ftarget",- )- response = connection.getresponse()- self.assertEqual(response.status, 307)- self.assertEqual(response.headers.get("Location"), "/target")- self.assertEqual(response.headers.get("Timing-Allow-Origin"), "*")- connection.close()-- def test_fixture_server_drains_xhr_delay_yo_body_before_next_request(- self,- ) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- server = WptFixtureServer(root_path)- server.httpd.RequestHandlerClass.protocol_version = "HTTP/1.1"- with server, patch(- "moli_benchmark.wpt_cross.server.time.sleep"- ):- connection = HTTPConnection("localhost", server.port, timeout=2)- try:- path = "/xhr/resources/delay.py?ms=0"- connection.request("YO", path, body=b"upload")- first = connection.getresponse()- self.assertEqual(- (first.status, first.read()),- (200, b"TEST_DELAY"),- )- first_socket = connection.sock-- connection.request("GET", path)- second = connection.getresponse()- self.assertEqual(- (second.status, second.read()),- (200, b"TEST_DELAY"),- )- self.assertIs(connection.sock, first_socket)- finally:- connection.close()-- def test_fixture_server_drains_chunked_xhr_delay_body_before_next_request(- self,- ) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- server = WptFixtureServer(root_path)- server.httpd.RequestHandlerClass.protocol_version = "HTTP/1.1"- with server, patch(- "moli_benchmark.wpt_cross.server.time.sleep"- ):- connection = HTTPConnection("localhost", server.port, timeout=2)- try:- path = "/xhr/resources/delay.py?ms=0"- connection.request(- "POST",- path,- body=[b"chunk-one", b"chunk-two"],- encode_chunked=True,- )- first = connection.getresponse()- self.assertEqual(- (first.status, first.read()),- (200, b"TEST_DELAY"),- )- first_socket = connection.sock-- connection.request("GET", path)- second = connection.getresponse()- self.assertEqual(- (second.status, second.read()),- (200, b"TEST_DELAY"),- )- self.assertIs(connection.sock, first_socket)- finally:- connection.close()-- def test_fixture_server_parses_wpt_header_pipe(self) -> None:- def applied_pipe_headers(query: str) -> list[tuple[str, str]]:- return _apply_header_operations(- [], _pipe_response_header_operations(query)- )-- self.assertEqual(- applied_pipe_headers(- "pipe=header(Access-Control-Allow-Origin,*)|header(X-Test,ok)"- ),- [("Access-Control-Allow-Origin", "*"), ("X-Test", "ok")],- )- self.assertEqual(- applied_pipe_headers(- "pipe=header(Content-Security-Policy,first)"- "|header(Content-Security-Policy,second,True)"- ),- [- ("Content-Security-Policy", "first"),- ("Content-Security-Policy", "second"),- ],- )- self.assertEqual(- applied_pipe_headers(- "pipe=header(Content-Security-Policy,first)"- "|header(Content-Security-Policy,second,False)"- ),- [("Content-Security-Policy", "second")],- )- self.assertEqual(applied_pipe_headers("pipe=header(Bad Header,ok)"), [])- self.assertEqual(- applied_pipe_headers("pipe=header(X-Test,bad%0D%0AInjected:%20x)"),- [("X-Test", "bad Injected: x")],- )- self.assertEqual(applied_pipe_headers("notpipe=header(X,Y)"), [])-- def test_fixture_server_parses_wpt_status_pipe(self) -> None:- self.assertEqual(_pipe_response_status("pipe=status(204)&cachebust=1"), 204)- self.assertEqual(- _pipe_response_status("pipe=header(X,Y)|status(205)"),- 205,- )- self.assertEqual(- _pipe_response_status("pipe=status(204)&pipe=status(205)"),- 205,- )- self.assertIsNone(_pipe_response_status("pipe=status(099)"))- self.assertIsNone(_pipe_response_status("pipe=status(600)"))- self.assertIsNone(_pipe_response_status("notpipe=status(204)"))-- def test_fixture_server_parses_wpt_headers_sidecar(self) -> None:- with tempfile.TemporaryDirectory() as root:- fixture = Path(root) / "case.html"- fixture.write_text("<!doctype html>", encoding="utf-8")- fixture.with_name("case.html.headers").write_text(- "Referrer-Policy: no-referrer\n"- "Bad Header: skip\n"- "X-Test: ok\n",- encoding="utf-8",- )-- self.assertEqual(- _sidecar_response_headers(fixture),- [("Referrer-Policy", "no-referrer"), ("X-Test", "ok")],- )-- def test_fixture_server_parses_sub_headers_sidecar_for_sub_files(self) -> None:- with tempfile.TemporaryDirectory() as root:- fixture = Path(root) / "worker.sub.js"- fixture.write_text("// worker", encoding="utf-8")- fixture.with_name("worker.sub.js.sub.headers").write_text(- "Content-Security-Policy: connect-src 'none'\n",- encoding="utf-8",- )-- self.assertEqual(- _sidecar_response_headers(fixture),- [("Content-Security-Policy", "connect-src 'none'")],- )-- def test_fixture_server_combines_immediate_directory_and_file_headers(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- fixture_dir = root_path / "support"- fixture_dir.mkdir()- fixture = fixture_dir / "ufoo"- fixture.write_text("ufoo", encoding="utf-8")- fixture_dir.joinpath("__dir__.headers").write_text(- "Content-Type: text/html\nX-Directory: immediate\n",- encoding="utf-8",- )- fixture.with_name("ufoo.headers").write_text(- "X-File: exact\n",- encoding="utf-8",- )- root_path.joinpath("__dir__.headers").write_text(- "X-Directory: parent\n",- encoding="utf-8",- )-- self.assertEqual(- _sidecar_response_headers(fixture),- [- ("Content-Type", "text/html"),- ("X-Directory", "immediate"),- ("X-File", "exact"),- ],- )-- def test_fixture_server_prefers_file_sub_headers_when_both_exist(self) -> None:- with tempfile.TemporaryDirectory() as root:- fixture = Path(root) / "case.sub.html"- fixture.write_text("<!doctype html>", encoding="utf-8")- fixture.with_name("case.sub.html.headers").write_text(- "X-Source: plain\n",- encoding="utf-8",- )- fixture.with_name("case.sub.html.sub.headers").write_text(- "X-Source: substituted\n",- encoding="utf-8",- )-- self.assertEqual(- _sidecar_response_headers(fixture),- [("X-Source", "substituted")],- )-- def test_fixture_server_prefers_directory_sub_headers_when_both_exist(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- fixture = root_path / "case.html"- fixture.write_text("<!doctype html>", encoding="utf-8")- root_path.joinpath("__dir__.headers").write_text(- "X-Source: plain\n",- encoding="utf-8",- )- root_path.joinpath("__dir__.sub.headers").write_text(- "X-Source: substituted\n",- encoding="utf-8",- )-- self.assertEqual(- _sidecar_response_headers(fixture),- [("X-Source", "substituted")],- )-- def test_fixture_server_content_type_sidecar_overrides_static_guess(self) -> None:- content_type, headers = _response_content_type_and_extra_headers(- "application/octet-stream",- [- ("Content-Type", "text/javascript; charset=utf-8"),- ("X-Test", "ok"),- ],- )-- self.assertEqual(content_type, "text/javascript; charset=utf-8")- self.assertEqual(headers, [("X-Test", "ok")])-- def test_fixture_server_combines_sidecar_and_pipe_headers_for_static_responses(- self,- ) -> None:- with tempfile.TemporaryDirectory() as root:- fixture = Path(root) / "feature.any.js"- fixture.write_text("// test", encoding="utf-8")- fixture.with_name("feature.any.js.headers").write_text(- "Content-Security-Policy: script-src 'self'\n",- encoding="utf-8",- )-- self.assertEqual(- _static_response_headers(fixture, "pipe=header(X-Test,ok)"),- [- ("Content-Security-Policy", "script-src 'self'"),- ("X-Test", "ok"),- ],- )- self.assertEqual(- _static_response_headers(- fixture,- "pipe=header(Content-Security-Policy,script-src 'none',False)",- ),- [("Content-Security-Policy", "script-src 'none'")],- )- self.assertEqual(- _static_response_headers(- fixture,- "pipe=header(Content-Security-Policy,require-trusted-types-for 'script',True)",- ),- [- ("Content-Security-Policy", "script-src 'self'"),- (- "Content-Security-Policy",- "require-trusted-types-for 'script'",- ),- ],- )-- def test_fixture_server_reads_sub_headers_for_plain_static_resource(self) -> None:- with tempfile.TemporaryDirectory() as root:- fixture = Path(root) / "policy.html"- fixture.write_text("<!doctype html>", encoding="utf-8")- fixture.with_name("policy.html.sub.headers").write_text(- "Set-Cookie: policy={{$id:uuid()}}\n"- "Content-Security-Policy: frame-src 'none'; report-uri /report.py?reportID={{$id}}\n",- encoding="utf-8",- )-- headers = _static_response_headers(fixture, "", port=8000)-- self.assertEqual(len(headers), 2)- cookie_value = headers[0][1].removeprefix("policy=")- self.assertEqual(headers[0][0], "Set-Cookie")- self.assertNotIn("{{", cookie_value)- self.assertEqual(- headers[1],- (- "Content-Security-Policy",- f"frame-src 'none'; report-uri /report.py?reportID={cookie_value}",- ),- )-- def test_fixture_server_models_reporting_resource_stash(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- with WptFixtureServer(root_path) as server:- report_url = f"{server.base_url}/reporting/resources/report.py?op=put&reportID=abc"- payload = json.dumps(- {"csp-report": {"violated-directive": "frame-src 'none'"}}- ).encode("utf-8")- request = Request(- report_url,- data=payload,- headers={"Content-Type": "application/csp-report"},- method="POST",- )- with urlopen(request, timeout=2) as response:- self.assertEqual(response.status, 200)-- retrieve_url = (- f"{server.base_url}/reporting/resources/report.py"- "?op=retrieve_report&timeout=0&reportID=abc"- )- with urlopen(retrieve_url, timeout=2) as response:- reports = json.loads(response.read().decode("utf-8"))-- self.assertEqual(- reports[0]["csp-report"]["violated-directive"],- "frame-src 'none'",- )- self.assertEqual(- reports[0]["metadata"]["content_type"],- "application/csp-report",- )-- def test_fixture_server_accepts_standard_websocket_echo_endpoint(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- with WptFixtureServer(root_path) as server:- with socket.create_connection(- ("127.0.0.1", server.port), timeout=2- ) as connection:- connection.sendall(- b"GET /echo HTTP/1.1\r\n"- b"Host: localhost\r\n"- b"Connection: Upgrade\r\n"- b"Upgrade: websocket\r\n"- b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"- b"Sec-WebSocket-Version: 13\r\n\r\n"- )- response = b""- while b"\r\n\r\n" not in response:- response += connection.recv(4096)- self.assertTrue(response.startswith(b"HTTP/1.0 101"))- self.assertIn(b"Upgrade: websocket\r\n", response)- self.assertIn(- b"Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n",- response,- )-- mask = b"\x01\x02\x03\x04"- payload = b"ping"- masked = bytes(- value ^ mask[index % len(mask)]- for index, value in enumerate(payload)- )- connection.sendall(b"\x81\x84" + mask + masked)- echoed = b""- while len(echoed) < 6:- echoed += connection.recv(6 - len(echoed))- self.assertEqual(echoed, b"\x81\x04ping")-- def test_fixture_server_serves_raw_asis_header_bytes(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- (root_path / "raw.asis").write_bytes(- b"HTTP/1.1 200 OK\r\n"- b"Content-Type: text/plain\r\n"- b"X-Custom-Header-Bytes: \xe2\x80\xa6\r\n"- b"\r\n"- b"OK"- )-- with WptFixtureServer(root_path) as server:- with urlopen(f"{server.base_url}/raw.asis", timeout=2) as response:- self.assertEqual(response.read(), b"OK")- header = response.headers["X-Custom-Header-Bytes"]-- self.assertEqual(header.encode("latin-1"), b"\xe2\x80\xa6")-- def test_fixture_server_serves_raw_sidecar_header_bytes(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- (root_path / "manifest.webmanifest").write_bytes(b"{}")- (root_path / "manifest.webmanifest.headers").write_bytes(- b"Content-Type: \xc3\x97\xc2\xba invalid\r\n"- )-- with WptFixtureServer(root_path) as server:- with urlopen(- f"{server.base_url}/manifest.webmanifest",- timeout=2,- ) as response:- self.assertEqual(response.read(), b"{}")- header = response.headers["Content-Type"]-- self.assertEqual(header.encode("latin-1"), b"\xc3\x97\xc2\xba invalid")-- def test_fixture_server_uses_explicit_content_type_header_as_override(self) -> None:- self.assertEqual(- _static_response_header_block(- "application/octet-stream",- [("Content-Type", "application/wasm"), ("X-Test", "ok")],- ),- [("Content-Type", "application/wasm"), ("X-Test", "ok")],- )- self.assertEqual(- _static_response_header_block("application/javascript", [("X-Test", "ok")]),- [("Content-Type", "application/javascript"), ("X-Test", "ok")],- )-- def test_fixture_server_detects_explicit_content_length_header(self) -> None:- header_block = _static_response_header_block(- "text/html",- [("Content-Length", "403"), ("X-Test", "ok")],- )-- self.assertTrue(_headers_include(header_block, "content-length"))-- def test_fixture_server_preserves_explicit_content_length_body_boundary(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness",- encoding="utf-8",- )- body = (- b'<!doctype html><script src="/resources/testharnessreport.js"></script>'- b"<body>PASS"- )- fixture = root_path / "content-length.html"- fixture.write_bytes(body + b"FAIL")- fixture.with_name("content-length.html.headers").write_text(- f"Content-Length: {len(body)}\n",- encoding="utf-8",- )-- with WptFixtureServer(root_path) as server:- server.set_harness_timeout_multipliers(- {"content-length.html": 12.0}- )- with urlopen(- f"{server.base_url}/content-length.html",- timeout=2,- ) as response:- served = response.read()-- self.assertEqual(served, body)- self.assertNotIn(BENCH_TIMEOUT_MULTIPLIER_QUERY.encode("ascii"), served)-- def test_fixture_server_maps_legacy_webidl_parser_resource(self) -> None:- self.assertEqual(- _legacy_wpt_resource_alias("/resources/WebIDLParser.js"),- "resources/webidl2/lib/webidl2.js",- )- self.assertIsNone(_legacy_wpt_resource_alias("/resources/testharness.js"))-- def test_fixture_server_builds_any_js_window_wrapper(self) -> None:- body = _any_js_window_wrapper(- "/WebCryptoAPI/sign_verify/hmac.https.any.html",- b"// META: title=WebCryptoAPI hmac\n"- b"// META: script=../util/helpers.js\n"- b"// META: script=hmac_vectors.js\n"- b"// META: timeout=long\n"- b"run_test();\n",- )-- self.assertIn(b'<meta name="timeout" content="long">', body)- self.assertIn(b"<title>WebCryptoAPI hmac</title>", body)- self.assertIn(b"self.GLOBAL = {", body)- self.assertIn(b'<script src="/resources/testharness.js"></script>', body)- self.assertIn(b'<script src="/resources/testharnessreport.js"></script>', body)- self.assertIn(b'<script src="../util/helpers.js"></script>', body)- self.assertIn(b'<script src="hmac_vectors.js"></script>', body)- self.assertIn(b'<div id="log"></div>', body)- self.assertIn(b'<script src="hmac.https.any.js"></script>', body)-- def test_fixture_server_builds_window_js_wrapper_without_any_global(self) -> None:- body = _window_js_window_wrapper(- "/WebCryptoAPI/algorithm-discards-context.https.window.html",- b"// META: title=Window case\n"- b"// META: script=helper.js\n"- b"test(() => {}, 'ok');\n",- )-- self.assertIn(b"<title>Window case</title>", body)- self.assertIn(b'<script src="/resources/testharness.js"></script>', body)- self.assertIn(b'<script src="/resources/testharnessreport.js"></script>', body)- self.assertIn(b'<script src="helper.js"></script>', body)- self.assertIn(b'<script src="algorithm-discards-context.https.window.js"></script>', body)- self.assertNotIn(b"self.GLOBAL", body)-- def test_fixture_server_builds_wpt_any_js_query_wrapper(self) -> None:- html = _wpt_any_window_wrapper_html(- "wasm/jsapi/feature.any.js",- "// META: script=../support/helper.js\n// META: script=/common/gc.js?run=1\n",- query="variant=1&moli-wpt-any=window",- )-- self.assertIn('<script src="/resources/testharness.js"></script>', html)- self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)- self.assertIn('<script src="/wasm/support/helper.js"></script>', html)- self.assertIn('<script src="/common/gc.js?run=1"></script>', html)- self.assertIn(- '<script src="/wasm/jsapi/feature.any.js?variant=1"></script>',- html,- )- self.assertNotIn("moli-wpt-any", html)-- def test_fixture_server_builds_any_js_dedicated_worker_wrapper(self) -> None:- html = _wpt_any_dedicated_worker_wrapper_html(- "wasm/jsapi/feature.any.js",- query="variant=1&moli-wpt-any=dedicatedworker",- )-- self.assertIn('<script src="/resources/testharness.js"></script>', html)- self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)- self.assertIn(- 'fetch_tests_from_worker(new Worker("/wasm/jsapi/feature.any.worker.js?variant=1"))',- html,- )- self.assertNotIn("moli-wpt-any", html)-- def test_fixture_server_builds_any_js_dedicated_worker_script(self) -> None:- js = _wpt_any_dedicated_worker_wrapper_js(- "wasm/jsapi/feature.any.js",- "// META: script=../support/helper.js\n// META: script=/common/gc.js?run=1\n",- query="variant=1&moli-wpt-any=dedicatedworker",- )-- self.assertIn("isWindow:function(){return false;}", js)- self.assertIn("isWorker:function(){return true;}", js)- self.assertIn('importScripts("/resources/testharness.js");', js)- self.assertIn('importScripts("/wasm/support/helper.js");', js)- self.assertIn('importScripts("/common/gc.js?run=1");', js)- self.assertIn('importScripts("/wasm/jsapi/feature.any.js?variant=1");', js)- self.assertTrue(js.rstrip().endswith("done();"))- self.assertNotIn("moli-wpt-any", js)-- def test_fixture_server_builds_window_js_wrapper(self) -> None:- html = _wpt_window_js_wrapper_html(- "wasm/serialization/transfer.window.js",- "// META: script=../support/helper.js\n// META: script=/common/gc.js?run=1\n",- query="variant=1&moli-wpt-script=window",- )-- self.assertIn('<script src="/resources/testharness.js"></script>', html)- self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)- self.assertIn('<script src="/wasm/support/helper.js"></script>', html)- self.assertIn('<script src="/common/gc.js?run=1"></script>', html)- self.assertIn(- '<script src="/wasm/serialization/transfer.window.js?variant=1"></script>',- html,- )- self.assertNotIn("moli-wpt-script", html)-- def test_fixture_server_builds_worker_js_wrapper(self) -> None:- html = _wpt_dedicated_worker_js_wrapper_html(- "wasm/create_multiple_memory.worker.js",- query="variant=1&moli-wpt-script=dedicatedworker",- )-- self.assertIn('<script src="/resources/testharness.js"></script>', html)- self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)- self.assertIn(- 'fetch_tests_from_worker(new Worker("/wasm/create_multiple_memory.worker.js?variant=1"))',- html,- )- self.assertNotIn("moli-wpt-script", html)-- def test_any_js_worker_script_path_round_trips_to_source_path(self) -> None:- worker_path = any_js_worker_script_path(- "wasm/jsapi/feature.any.js?variant=1"- )-- self.assertEqual(- worker_path,- "wasm/jsapi/feature.any.worker.js?variant=1",- )- self.assertEqual(- any_js_source_script_path(worker_path),- "wasm/jsapi/feature.any.js?variant=1",- )-- def test_any_js_case_path_for_global_replaces_existing_wrapper_query(self) -> None:- self.assertEqual(- any_js_case_path_for_global(- "wasm/jsapi/feature.any.js?moli-wpt-any=window&variant=1",- ANY_JS_DEDICATED_WORKER_GLOBAL,- ),- "wasm/jsapi/feature.any.js?variant=1&moli-wpt-any=dedicatedworker",- )-- def test_fixture_server_resolves_any_js_meta_scripts_within_wpt_root(self) -> None:- self.assertEqual(- _resolve_wpt_static_script_url(- "wasm/jsapi/feature.any.js",- "../support/helper.js?mode=1",- ),- "/wasm/support/helper.js?mode=1",- )- self.assertEqual(- _resolve_wpt_static_script_url(- "wasm/jsapi/feature.any.js",- "/resources/WebIDLParser.js",- ),- "/resources/WebIDLParser.js",- )- self.assertIsNone(- _resolve_wpt_static_script_url(- "wasm/jsapi/feature.any.js",- "../../../escape.js",- )- )-- def test_fixture_server_substitutes_core_sub_template_variables(self) -> None:- body = (- b"http://{{domains[www2]}}:{{ports[http][0]}}/"- b" location={{location[port]}}"- b" scheme={{location[scheme]}}"- b" hostname={{location[hostname]}}"- b" server={{location[server]}}"- b" path={{location[path]}}"- b" alt={{ports[http][1]}}"- b" same={{domains[www]}} host={{host}}"- b" hosts={{hosts[][]}}/{{hosts[][www]}}/{{hosts[alt][]}}/{{hosts[alt][www]}}"- b" https={{ports[https][0]}}/{{ports[https][1]}}"- b" https-url=https://{{domains[www]}}:{{ports[https][0]}}/secure/"- b" https-remote=https://{{domains[www1]}}:{{ports[https][0]}}/remote/"- b" https-hosts-remote=https://{{hosts[][www]}}:{{ports[https][1]}}/cross/"- b" https-hosts-bare=https://{{hosts[][]}}:{{ports[https][0]}}/bare/"- b" https-hosts-alt=https://{{hosts[alt][]}}:{{ports[https][0]}}/alt/"- b" https-hosts-alt-www=https://{{hosts[alt][www]}}:{{ports[https][1]}}/alt-www/"- b" ws={{ports[ws][0]}}/{{ports[ws][1]}}"- b" wss={{ports[wss][0]}}/{{ports[wss][1]}}"- b" ws-url=ws://{{host}}:{{ports[ws][0]}}/socket"- b" wss-url=wss://{{host}}:{{ports[wss][0]}}/socket"- )-- self.assertEqual(- _substitute_wpt_template_variables(- body,- port=12345,- alternate_port=23456,- request_path="/secure-contexts/server-locations.sub.js",- request_hostname="example.test",- ),- b"http://www2.localhost:12345/ location=12345 scheme=http hostname=example.test"- b" server=http://example.test:12345 path=/secure-contexts/server-locations.sub.js"- b" alt=23456 same=www.localhost host=example.test"- b" hosts=localhost/www.localhost/alt.localhost/www.alt.localhost https=12345/23456"- b" https-url=http://example.test:12345/secure/"- b" https-remote=http://example.test:23456/remote/"- b" https-hosts-remote=http://www.localhost:23456/cross/"- b" https-hosts-bare=http://localhost:12345/bare/"- b" https-hosts-alt=http://alt.localhost:12345/alt/"- b" https-hosts-alt-www=http://www.alt.localhost:23456/alt-www/"- b" ws=12345/23456 wss=12345/23456"- b" ws-url=ws://example.test:12345/socket"- b" wss-url=ws://example.test:12345/socket",- )-- def test_fixture_server_pipe_sub_requests_template_substitution(self) -> None:- self.assertTrue(- _needs_wpt_template_substitution(- "sandboxed-tests.html",- b'importScripts("http://{{host}}:{{ports[http][0]}}/resources/testharness.js");',- "pipe=sub",- )- )- self.assertTrue(- _needs_wpt_template_substitution(- "resource.html",- b"{{host}}",- "pipe=header(X-Test,yes)|sub",- )- )- self.assertFalse(- _needs_wpt_template_substitution(- "resource.html",- b"{{host}}",- "",- )- )-- def test_fixture_server_maps_external_ipv6_domain_location_port_to_remote_port(- self,- ) -> None:- body = (- b"http://{{domains[www]}}:{{location[port]}}/a"- b" http://{{domains[www2]}}:{{location[port]}}/b"- b" http://{{domains[www]}}:{{ports[http][0]}}/c"- b" http://{{domains[www1]}}:{{ports[http][0]}}/d"- b" http://{{domains[www1]}}:{{ports[http][1]}}/e"- )-- self.assertEqual(- _substitute_wpt_template_variables(- body,- port=12345,- alternate_port=23456,- remote_port=34567,- request_hostname="[2001:db8::1]",- ),- b"http://[2001:db8::1]:34567/a"- b" http://[2001:db8::1]:34567/b"- b" http://[2001:db8::1]:34567/c"- b" http://[2001:db8::1]:34567/d"- b" http://[2001:db8::1]:23456/e",- )-- def test_fixture_server_substitutes_get_query_template_variables(self) -> None:- body = (- b"var expected_logs = {{GET[logs]}};"- b" var timeout = \"{{GET[timeout]}}\";"- b" var missing = \"{{GET[missing]}}\";"- )-- self.assertEqual(- _substitute_wpt_template_variables(- body,- port=12345,- query='logs=["xhr allowed","TEST COMPLETE"]&timeout=2',- ),- b'var expected_logs = ["xhr allowed","TEST COMPLETE"];'- b' var timeout = "2";'- b' var missing = "";',- )-- def test_fixture_server_substitutes_template_variables_in_sidecar_headers(self) -> None:- with tempfile.TemporaryDirectory() as root:- fixture = Path(root) / "frame-ancestors.sub.html"- fixture.write_text("<!doctype html>", encoding="utf-8")- fixture.with_name("frame-ancestors.sub.html.sub.headers").write_text(- "Content-Security-Policy: frame-ancestors {{GET[policy]}} {{location[scheme]}}://{{location[host]}}\n",- encoding="utf-8",- )-- self.assertEqual(- _static_response_headers(- fixture,- "policy=%27none%27",- port=12345,- alternate_port=23456,- request_hostname="example.test",- ),- [- (- "Content-Security-Policy",- "frame-ancestors 'none' http://example.test:12345",- )- ],- )-- def test_fixture_server_substitution_preserves_non_utf8_bytes(self) -> None:- body = b"\xff{{host}}\xfe{{ports[http][0]}}"-- self.assertEqual(- _substitute_wpt_template_variables(body, port=12345),- b"\xfflocalhost\xfe12345",- )-- def test_fixture_server_distinguishes_primary_host_from_request_hostname(- self,- ) -> None:- body = (- b"host={{host}} "- b"location={{location[hostname]}} "- b"HTTP_ORIGIN: 'http://' + ORIGINAL_HOST + HTTP_PORT_ELIDED,"- )-- self.assertEqual(- _substitute_wpt_template_variables(- body,- port=12345,- request_hostname="www1.localhost",- primary_hostname="localhost",- ),- b"host=localhost location=www1.localhost "- b"HTTP_ORIGIN: 'http://' + ORIGINAL_HOST + HTTP_PORT_ELIDED,",- )-- def test_host_header_hostname_preserves_ipv6_brackets(self) -> None:- self.assertEqual(_host_header_hostname("[2001:db8::1]:1234"), "[2001:db8::1]")- self.assertEqual(_host_header_hostname("localhost:1234"), "localhost")- self.assertEqual(_host_header_hostname("example.test"), "example.test")-- def test_fixture_server_substitutes_ipv6_websocket_template_urls(self) -> None:- body = (- b"ws=ws://{{host}}:{{ports[ws][0]}}/echo"- b" wss=wss://{{host}}:{{ports[wss][0]}}/echo"- )-- self.assertEqual(- _substitute_wpt_template_variables(- body,- port=12345,- request_hostname="[2001:db8::1]",- ),- b"ws=ws://[2001:db8::1]:12345/echo"- b" wss=ws://[2001:db8::1]:12345/echo",- )-- def test_fixture_server_wasm_status_handler_normalizes_status_codes(self) -> None:- self.assertEqual(_wasm_webapi_status_code("status=404"), 404)- self.assertEqual(_wasm_webapi_status_code("status=300"), 300)- self.assertEqual(_wasm_webapi_status_code("status=0"), 599)- self.assertEqual(_wasm_webapi_status_code("status=700"), 599)- self.assertEqual(_wasm_webapi_status_code("status=not-a-number"), 400)- self.assertEqual(_wasm_webapi_status_code(""), 200)-- def test_fixture_server_redirect_handler_models_wasm_origin_probe(self) -> None:- self.assertEqual(- _redirect_fixture_response(- "redirect_status=301&location=/wasm/incrementer.wasm"- ),- (301, "/wasm/incrementer.wasm"),- )- self.assertIsNone(_redirect_fixture_response("redirect_status=200&location=/x"))- self.assertIsNone(_redirect_fixture_response("redirect_status=301"))- self.assertIsNone(- _redirect_fixture_response("redirect_status=301&location=/x%0Dbad")- )- self.assertEqual(- _redirect_fixture_response("status=307&location=/target"),- (307, "/target"),- )-- def test_fixture_server_models_csp_resource_py(self) -> None:- body, headers = _content_security_policy_resource_response()-- self.assertIn(b"success", body)- self.assertIn(("Access-Control-Allow-Origin", "*"), headers)-- def test_fixture_server_models_workers_modules_export_on_load_script_py(self) -> None:- body, headers = _workers_modules_export_on_load_script_response()-- self.assertIn(b"export const importedModules", body)- self.assertIn(("Content-Type", "text/javascript"), headers)- self.assertIn(("Access-Control-Allow-Origin", "*"), headers)-- def test_fixture_server_preserves_raw_sidecar_header_bytes(self) -> None:- with tempfile.TemporaryDirectory() as tmp:- root = Path(tmp)- resource = root / "resource.webmanifest"- resource.write_bytes(b"{}")- resource.with_name(resource.name + ".headers").write_bytes(- b"Content-Type: \xc3\x97\xc2\xba invalid\n"- )-- headers = _sidecar_response_headers(resource)-- self.assertEqual(headers, [("Content-Type", "\xc3\x97\xc2\xba invalid")])- for _name, value in headers:- value.encode("latin-1")-- def test_fixture_server_substitutes_get_host_info_remote_host_for_loopback(self) -> None:- body = (- b"var REMOTE_HOST = (ORIGINAL_HOST === 'localhost') ? "- b"'127.0.0.1' : ('www1.' + ORIGINAL_HOST);"- b"\nHTTP_REMOTE_ORIGIN: 'http://' + REMOTE_HOST + HTTP_PORT_ELIDED,"- b"\nREMOTE_ORIGIN: PROTOCOL + \"//\" + REMOTE_HOST + PORT_ELIDED,"- b"\nOTHER_ORIGIN: PROTOCOL + \"//\" + OTHER_HOST + PORT_ELIDED,"- b"\nHTTP_NOTSAMESITE_ORIGIN: 'http://' + NOTSAMESITE_HOST + HTTP_PORT_ELIDED,"- b"\nHTTPS_ORIGIN: 'https://' + ORIGINAL_HOST + HTTPS_PORT_ELIDED,"- b"\nHTTPS_ORIGIN_WITH_CREDS: 'https://foo:bar@' + ORIGINAL_HOST + HTTPS_PORT_ELIDED,"- b"\nHTTPS_REMOTE_ORIGIN: 'https://' + REMOTE_HOST + HTTPS_PORT_ELIDED,"- b"\nHTTPS_REMOTE_ORIGIN_WITH_CREDS: 'https://foo:bar@' + REMOTE_HOST + HTTPS_PORT_ELIDED,"- )-- self.assertEqual(- _substitute_wpt_template_variables(- body,- port=12345,- alternate_port=23456,- remote_port=34567,- ),- b"var REMOTE_HOST = (ORIGINAL_HOST === 'localhost') ? "- b"'www1.localhost' : ((ORIGINAL_HOST.indexOf(':') !== -1) ? "- b"ORIGINAL_HOST : ('www1.' + ORIGINAL_HOST));"- b"\nHTTP_REMOTE_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "- b"('http://' + REMOTE_HOST + ':23456') : ('http://' + REMOTE_HOST + HTTP_PORT_ELIDED),"- b"\nREMOTE_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "- b"('http://' + REMOTE_HOST + ':23456') : (PROTOCOL + \"//\" + REMOTE_HOST + PORT_ELIDED),"- b"\nOTHER_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "- b"('http://' + ORIGINAL_HOST + ':34567') : (PROTOCOL + \"//\" + OTHER_HOST + PORT_ELIDED),"- b"\nHTTP_NOTSAMESITE_ORIGIN: 'http://' + NOTSAMESITE_HOST + HTTP_PORT_ELIDED,"- b"\nHTTPS_ORIGIN: 'http://' + ORIGINAL_HOST + HTTP_PORT2_ELIDED,"- b"\nHTTPS_ORIGIN_WITH_CREDS: 'http://foo:bar@' + ORIGINAL_HOST + HTTP_PORT2_ELIDED,"- b"\nHTTPS_REMOTE_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "- b"('http://' + REMOTE_HOST + ':34567') : ('http://' + REMOTE_HOST + HTTP_PORT2_ELIDED),"- b"\nHTTPS_REMOTE_ORIGIN_WITH_CREDS: (ORIGINAL_HOST.indexOf(':') !== -1) ? "- b"('http://foo:bar@' + REMOTE_HOST + ':34567') : "- b"('http://foo:bar@' + REMOTE_HOST + HTTP_PORT2_ELIDED),",- )-- def test_cli_runner_distinguishes_app_errors_from_process_crashes(self) -> None:- self.assertEqual(_nonzero_exit_status(1, "Error: unsupported navigation"), "error")- self.assertEqual(_nonzero_exit_status(-11, ""), "crash")- self.assertEqual(_nonzero_exit_status(101, "thread 'main' panicked at x"), "crash")-- def test_cli_runner_stderr_tail_is_bounded(self) -> None:- self.assertEqual(_stderr_tail("short"), "short")- self.assertEqual(len(_stderr_tail("x" * 3000)), 2000)-- def test_cli_runner_uses_longer_payload_grace_only_after_successful_process(self) -> None:- self.assertEqual(- _payload_grace_for_process_result(- proc_error=None,- proc_returncode=0,- payload_grace_seconds=2.0,- successful_process_payload_grace_seconds=8.0,- ),- 8.0,- )- self.assertEqual(- _payload_grace_for_process_result(- proc_error="engine subprocess wall timeout",- proc_returncode=None,- payload_grace_seconds=2.0,- successful_process_payload_grace_seconds=5.0,- ),- 2.0,- )- self.assertEqual(- _payload_grace_for_process_result(- proc_error=None,- proc_returncode=1,- payload_grace_seconds=2.0,- successful_process_payload_grace_seconds=5.0,- ),- 2.0,- )-- def test_cli_runner_keeps_final_payload_status_after_process_timeout(self) -> None:- class FakeResults:- def wait_for_final(self, key: str, timeout: float) -> dict:- self.wait_key = key- return {- "source": "completion-callback",- "harness": {"status": 0, "message": None},- "tests": [{"name": "done", "status": 0, "message": None}],- }-- def get(self, key: str) -> None:- return None-- fixture_server = SimpleNamespace(- results=FakeResults(),- )-- result = _classify_cli_case_result(- case_path="case.html",- url="http://example.test/case.html",- bridge_key="/case.html",- fixture_server=fixture_server,- subprocess_result=_CliSubprocessResult(- duration_ms=10.0,- proc_error="engine subprocess wall timeout after 0.0s",- proc_returncode=None,- proc_stderr="",- proc_stdout=b"",- wait_script_timeout=False,- ),- payload_grace_seconds=0,- successful_process_payload_grace_seconds=8.0,- )-- self.assertEqual(result.status, "pass")- self.assertIsNone(result.error)- self.assertEqual(result.payload_source, "completion-callback")-- def test_cli_runner_defaults_harness_timeout_independent_of_process_deadline(self) -> None:- class FakeFuture:- def __init__(self, result: CaseResult) -> None:- self._result = result-- def result(self) -> CaseResult:- return self._result-- class FakeProcessPoolExecutor:- instances: list["FakeProcessPoolExecutor"] = []-- def __init__(self, *, max_workers: int, mp_context: object) -> None:- self.max_workers = max_workers- self.mp_context = mp_context- self.submitted = []- self.instances.append(self)-- def __enter__(self) -> "FakeProcessPoolExecutor":- return self-- def __exit__(self, *args: object) -> None:- return None-- def submit(self, fn, job):- self.submitted.append((fn, job))- return FakeFuture(fn(job))-- def fake_as_completed(futures):- return list(futures)-- def fake_worker(job):- return CaseResult(- case_path=job.case_path,- url=f"http://worker.test/{job.case_path}",- status="pass",- duration_ms=1.0,- )-- driver = SimpleNamespace(- name="moli",- version_args=["--version"],- extra_env={},- cli_fetch_command=lambda binary, url, timeout: [str(binary), "fetch", url],- resolve_binary=lambda override: Path("/tmp/moli"),- )- fixture_server = SimpleNamespace(- external_host="2001:db8::1",- external_base_url="http://[2001:db8::1]:9000",- wpt_root=Path("/tmp/wpt"),- )-- def fake_subprocess_run(argv, **kwargs):- self.assertEqual(argv, ["/tmp/moli", "--version"])- return SimpleNamespace(stdout="moli 0\n", stderr="", returncode=0)-- with (- patch("moli_benchmark.wpt_cross.cli_runner.sha256_file", return_value="sha"),- patch("moli_benchmark.wpt_cross.cli_runner.subprocess.run", fake_subprocess_run),- patch(- "moli_benchmark.wpt_cross.cli_runner.ProcessPoolExecutor",- FakeProcessPoolExecutor,- ),- patch("moli_benchmark.wpt_cross.cli_runner.as_completed", fake_as_completed),- patch("moli_benchmark.wpt_cross.cli_runner._run_cli_case_worker", fake_worker),- ):- result = run_engine_on_cases_cli(- driver=driver,- fixture_server=fixture_server,- cases=[- (- "normal.html",- "http://[2001:db8::1]:9000/normal.html",- 120.0,- ),- (- "long.html",- "http://[2001:db8::1]:9000/long.html",- 120.0,- LONG_TIMEOUT_MULTIPLIER,- ),- ],- parallelism=1,- progress_every=0,- )-- pool = FakeProcessPoolExecutor.instances[0]- self.assertEqual(pool.max_workers, 1)- self.assertEqual(pool.mp_context.get_start_method(), "spawn")- self.assertEqual(pool.submitted[0][1].case_path, "normal.html")- self.assertTrue(pool.submitted[0][1].external)- self.assertEqual(pool.submitted[0][1].timeout_seconds, 120.0)- self.assertEqual(pool.submitted[0][1].harness_timeout_multiplier, 1.0)- self.assertEqual(- pool.submitted[1][1].harness_timeout_multiplier,- LONG_TIMEOUT_MULTIPLIER,- )- self.assertTrue(all(case.status == "pass" for case in result.cases))- self.assertEqual(result.shutdown_info["scheduler"], "process-pool")-- def test_moli_cli_worker_appends_hardcoded_wpt_user_agent(self) -> None:- captured_argv = []-- class FakeResults:- def clear(self, key: str) -> None:- return None-- def wait_for_final(self, key: str, timeout: float) -> dict:- return {- "source": "completion-callback",- "harness": {"status": 0, "message": None},- "tests": [{"name": "done", "status": 0, "message": None}],- }-- def get(self, key: str) -> None:- return None-- class FakeServer:- def __init__(self, wpt_root: Path) -> None:- self.wpt_root = Path(wpt_root)- self.results = FakeResults()- self.external_host = None-- def __enter__(self) -> "FakeServer":- return self-- def __exit__(self, *args: object) -> None:- return None-- def set_harness_timeout_multipliers(- self,- multipliers: dict[str, float],- *,- default_multiplier: float,- ) -> None:- return None-- def url_for_case(self, case_path: str, *, external: bool = False) -> str:- return f"http://127.0.0.1:8000/{case_path}"-- driver = SimpleNamespace(- name="moli",- cli_fetch_command=lambda binary, url, timeout: [str(binary), "fetch", url],- )-- def fake_run_cli_subprocess(argv, env, proc_timeout):- captured_argv.append(argv)- return _CliSubprocessResult(- duration_ms=1.0,- proc_error=None,- proc_returncode=0,- proc_stderr="",- proc_stdout=b"",- wait_script_timeout=False,- )-- with (- patch(- "moli_benchmark.wpt_cross.cli_runner.build_driver",- return_value=driver,- ),- patch(- "moli_benchmark.wpt_cross.cli_runner.WptFixtureServer",- FakeServer,- ),- patch(- "moli_benchmark.wpt_cross.cli_runner._run_cli_subprocess",- fake_run_cli_subprocess,- ),- ):- result = _run_cli_case_worker(- _CliCaseWorkerInput(- engine="moli",- binary="/tmp/moli",- wpt_root="/tmp/wpt",- case_path="case.html",- external=False,- timeout_seconds=8.0,- harness_timeout_multiplier=1.0,- env={},- process_timeout_margin_seconds=4.0,- payload_grace_seconds=0.0,- successful_process_payload_grace_seconds=0.0,- )- )-- self.assertEqual(result.status, "pass")- self.assertEqual(- captured_argv,- [- [- "/tmp/moli",- "fetch",- "http://127.0.0.1:8000/case.html",- "--user-agent",- MOLI_WPT_USER_AGENT,- ]- ],- )-- def test_moli_cli_resolves_external_fixture_template_hosts(self) -> None:- fixture_server = SimpleNamespace(- external_host="2001:db8::42",- external_port=12345,- external_alternate_port=23456,- external_remote_port=23456,- )-- args = _moli_fixture_host_resolve_args(fixture_server)-- self.assertIn(- "--http-host-resolve",- args,- )- self.assertIn(- "alt.localhost:12345:[2001:db8::42]",- args,- )- self.assertIn(- "www.localhost:23456:[2001:db8::42]",- args,- )- self.assertEqual(- args.count("localhost:23456:[2001:db8::42]"),- 1,- )-- def test_render_html_escapes_embedded_json_script_data(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir)- (output_dir / "summary.json").write_text(- json.dumps({"total": 1, "engines": {"moli": {"pass": 1}}}),- encoding="utf-8",- )- (output_dir / "matrix.json").write_text(- json.dumps(- [- {- "case_path": "xss.html",- "results": {- "moli": {- "status": "</script><img src=x onerror=alert(1)>",- "duration_ms": 1,- }- },- }- ]- ),- encoding="utf-8",- )-- html = render_html(output_dir).read_text(encoding="utf-8")-- self.assertNotIn("</script><img", html)- self.assertIn("\\u003c/script\\u003e", html)-- def test_render_html_includes_recorded_failure_drift(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir)- (output_dir / "summary.json").write_text(- json.dumps(- {- "total": 1,- "engines": {"moli": {"fail": 1}, "chrome": {"fail": 1}},- "recorded_failure_drift": {- "primary": "moli",- "recorded_failure_limit_per_engine": 40,- "comparison_count": 1,- "comparisons": [- {- "case_path": "WebCryptoAPI/shared.html",- "primary": "moli",- "peer": "chrome",- "primary_only_count": 1,- "peer_only_count": 0,- "message_diff_count": 0,- "primary_only_examples": ["lm-only"],- "peer_only_examples": [],- "message_diff_examples": [],- }- ],- },- }- ),- encoding="utf-8",- )- (output_dir / "matrix.json").write_text(- json.dumps(- [- {- "case_path": "WebCryptoAPI/shared.html",- "results": {- "moli": {"status": "fail", "duration_ms": 1},- "chrome": {"status": "fail", "duration_ms": 1},- },- }- ]- ),- encoding="utf-8",- )-- html = render_html(output_dir).read_text(encoding="utf-8")-- self.assertIn("Recorded subtest drift", html)- self.assertIn("lm-only", html)-- def test_render_html_includes_known_failure_audit_summary(self) -> None:- with tempfile.TemporaryDirectory() as temp_dir:- output_dir = Path(temp_dir)- (output_dir / "summary.json").write_text(- json.dumps(- {- "total": 1,- "engines": {"moli": {"fail": 1}},- "known_failure_audits": {- "moli": {- "ok": False,- "manifest": "known.json",- "output": "known-failure-audit-moli.json",- "counts": {- "known_failures": 1,- "resolved_known_failures": 1,- "mismatched_known_failures": 0,- "missing_expected_failures": 0,- "skipped_known_failures": 2,- "unexpected_failures": 0,- },- "category_counts": {- "known_failures": {- "wasm-global-live-binding": 1,- }- },- "categories": {- "wasm-global-live-binding": {- "tracking_doc": "docs/wasm-global-live-binding-design-current.md",- "scope": "V8-backed live binding work",- "evidence": [- {- "kind": "doc",- "path": "docs/wasm-global-live-binding-design-current.md",- "note": "local fixture evidence",- }- ],- }- },- }- },- }- ),- encoding="utf-8",- )- (output_dir / "matrix.json").write_text(- json.dumps(- [- {- "case_path": "wasm/example.html",- "results": {- "moli": {- "status": "fail",- "duration_ms": 1,- }- },- }- ]- ),- encoding="utf-8",- )-- html = render_html(output_dir).read_text(encoding="utf-8")-- self.assertIn("Known-failure audit", html)- self.assertIn("known-failure-audit-moli.json", html)- self.assertIn("wasm-global-live-binding", html)- self.assertIn("docs/wasm-global-live-binding-design-current.md", html)- self.assertIn("V8-backed live binding work", html)- self.assertIn("doc <code>docs/wasm-global-live-binding-design-current.md</code>", html)- self.assertIn("local fixture evidence", html)- self.assertIn("<th>skipped</th>", html)- self.assertIn("<td>2</td>", html)- self.assertIn(">attention<", html)-- def test_fixture_server_parses_fetch_status_parameters_as_bytes(self) -> None:- self.assertEqual(_fetch_status_response(""), (200, "OMG", "", b""))- self.assertEqual(- _fetch_status_response(- "code=201&code=404&text=&text=ignored&type=text%2Fplain"- "&content=%FF%FE%00%2B+end&content=ignored"- ),- (201, "", "text/plain", b"\xff\xfe\x00+ end"),- )-- def test_fixture_server_models_fetch_status_for_supported_methods(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text(- "// testharness", encoding="utf-8"- )- with WptFixtureServer(root_path) as server:- for method in ["GET", "HEAD", "POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO"]:- for status in [200, 204, 205, 304]:- with self.subTest(method=method, status=status):- connection = HTTPConnection("127.0.0.1", server.port, timeout=2)- try:- connection.request(- method,- "/fetch/api/resources/status.py"- f"?code={status}&text=Custom%20%FF"- "&type=text%2Fplain%3Bcharset%3DUTF-16"- "&content=%FF%FE%00%2B+end",- body=b"ignored" if method not in {"GET", "HEAD"} else None,- )- response = connection.getresponse()- self.assertEqual(response.status, status)- self.assertEqual(response.reason, "Custom \xff")- self.assertEqual(response.headers["X-Request-Method"], method)- self.assertEqual(- response.headers["Content-Type"],- "text/plain;charset=UTF-16",- )- self.assertEqual(response.headers["Content-Length"], "8")- self.assertEqual(- response.read(),- b"" if method == "HEAD" or status in {204, 304}- else b"\xff\xfe\x00+ end",- )- finally:- connection.close()-- def test_fixture_server_models_fetch_trickle_parameters_and_methods(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text("// testharness")- with WptFixtureServer(root_path) as server, patch(- "moli_benchmark.wpt_cross.server.time.sleep"- ) as sleep:- cases = [- ("", 50, "text/plain", 0.5),- ("count=2&count=9&ms=1.5&ms=900", 2, "text/plain", 0.0015),- ("count=0&ms=0¬ype", 0, None, 0.0),- ("count=-1&ms=0¬ype=false", 0, None, 0.0),- ]- for method in ["GET", "HEAD", "POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO"]:- for query, count, mime, delay in cases:- with self.subTest(method=method, query=query):- sleep.reset_mock()- connection = HTTPConnection("127.0.0.1", server.port, timeout=2)- try:- connection.request(- method, "/fetch/api/resources/trickle.py?" + query,- body=b"upload" if method not in {"GET", "HEAD"} else None,- )- response = connection.getresponse()- self.assertEqual(response.status, 200)- self.assertEqual(response.headers.get("Content-Type"), mime)- self.assertIsNone(response.headers.get("Content-Length"))- self.assertIsNone(response.headers.get("Transfer-Encoding"))- self.assertEqual(response.read(), b"" if method == "HEAD" else b"TEST_TRICKLE\n" * count)- expected_sleeps = 1 if method == "HEAD" else count + 2- self.assertEqual([args.args[0] for args in sleep.call_args_list], [delay] * expected_sleeps)- finally:- connection.close()- for query in ["count=invalid", "count=", "ms=invalid", "ms=-1", "ms=nan", "ms=inf"]:- with self.subTest(query=query):- connection = HTTPConnection("127.0.0.1", server.port, timeout=2)- try:- connection.request("GET", "/fetch/api/resources/trickle.py?" + query)- response = connection.getresponse()- self.assertEqual(response.status, 500)- response.read()- finally:- connection.close()-- def test_fixture_server_delivers_fetch_trickle_headers_and_chunks_before_eof(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text("// testharness")- headers_ready, chunk_ready = threading.Event(), threading.Event()- release_first, release_second = threading.Event(), threading.Event()- sleeps = []-- def pause(delay: float) -> None:- sleeps.append(delay)- if len(sleeps) == 2:- headers_ready.set()- self.assertTrue(release_first.wait(2))- elif len(sleeps) == 3:- chunk_ready.set()- self.assertTrue(release_second.wait(2))-- with WptFixtureServer(root_path) as server, patch(- "moli_benchmark.wpt_cross.server.time.sleep", side_effect=pause- ):- connection = HTTPConnection("127.0.0.1", server.port, timeout=2)- try:- connection.request("GET", "/fetch/api/resources/trickle.py?count=2¬ype")- response = connection.getresponse()- self.assertTrue(headers_ready.wait(2))- self.assertIsNone(response.headers.get("Content-Type"))- release_first.set()- self.assertEqual(response.read(13), b"TEST_TRICKLE\n")- self.assertTrue(chunk_ready.wait(2))- release_second.set()- self.assertEqual(response.read(), b"TEST_TRICKLE\n")- self.assertEqual(sleeps, [0.5] * 4)- finally:- release_first.set()- release_second.set()- connection.close()-- def test_fixture_server_reads_upload_before_starting_fetch_trickle(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text("// testharness")- started = threading.Event()- with WptFixtureServer(root_path) as server, patch(- "moli_benchmark.wpt_cross.server.time.sleep",- side_effect=lambda _: started.set(),- ):- for framing, first, last in [- (b"Content-Length: 6", b"abc", b"def"),- (b"Transfer-Encoding: chunked", b"3\r\nabc\r\n", b"3\r\ndef\r\n0\r\n\r\n"),- ]:- with self.subTest(framing=framing), socket.create_connection(("127.0.0.1", server.port), timeout=2) as connection:- started.clear()- connection.sendall(- b"POST /fetch/api/resources/trickle.py?count=1&ms=0 HTTP/1.1\r\n"- b"Host: localhost\r\n" + framing + b"\r\n\r\n" + first- )- self.assertFalse(started.wait(0.05), "response started before upload completed")- connection.sendall(last)- self.assertTrue(started.wait(2))- response = connection.makefile("rb").read()- self.assertEqual(response.split(b"\r\n\r\n", 1)[1], b"TEST_TRICKLE\n")---- def test_fixture_server_counts_link_stylesheet_requests_and_clears_on_read(self) -> None:- with tempfile.TemporaryDirectory() as root:- root_path = Path(root)- (root_path / "resources").mkdir()- (root_path / "resources" / "testharness.js").write_text("// testharness", encoding="utf-8")- resource = "/html/semantics/document-metadata/the-link-element/stylesheet.py"- fixture = root_path / resource.lstrip("/")- fixture.parent.mkdir(parents=True)- fixture.write_text("# dynamic stylesheet resource", encoding="utf-8")- first = "11111111-1111-4111-8111-111111111111"- second = "22222222-2222-4222-8222-222222222222"- with WptFixtureServer(root_path) as server:- def request(query: str, *, method: str = "GET", alternate: bool = False) -> tuple[str, bytes]:- port = server.alternate_port if alternate else server.port- url = f"http://127.0.0.1:{port}{resource}?{query}"- with urlopen(Request(url, method=method), timeout=2) as response:- self.assertEqual(response.status, 200)- self.assertIsNone(response.headers.get("Cache-Control"))- return response.headers["Content-Type"], response.read()-- self.assertEqual(request(f"id={first}&count="), ("text/html", b"0"))- self.assertEqual(request(f"id={first}"), ("text/css", b"body {color: red;}"))- self.assertEqual(request(f"id={first}", alternate=True), ("text/css", b"body {color: red;}"))- self.assertEqual(request(f"id={first}", method="HEAD"), ("text/css", b""))- self.assertEqual(request(f"id={second}"), ("text/css", b"body {color: red;}"))- self.assertEqual(request(f"id={first}&count=foo"), ("text/html", b"3"))- self.assertEqual(request(f"id={second}&count=foo"), ("text/html", b"1"))- self.assertEqual(request(f"id={first}&count=foo"), ("text/html", b"0"))- self.assertEqual(request(f"id={first}"), ("text/css", b"body {color: red;}"))- self.assertEqual(request(f"id={first}&count=foo"), ("text/html", b"1"))---if __name__ == "__main__":- unittest.main()diff --git a/moli-benchmark/tests/test_wpt_cross_case_configuration.py b/moli-benchmark/tests/test_wpt_cross_case_configuration.pynew file mode 100644index 000000000..7c9b82fce--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_case_configuration.py@@ -0,0 +1,391 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossCaseConfigurationTests(WptCrossTestCase):+ def test_moli_wpt_commands_enable_layout_and_resources(self) -> None:+ self.assertEqual(+ _moli_command(Path("/bin/moli"), 9222, None),+ [+ "/bin/moli",+ "serve",+ "--layout",+ "--resource",+ "--host",+ "127.0.0.1",+ "--port",+ "9222",+ ],+ )+ fetch = _moli_fetch(+ Path("/bin/moli"),+ "http://127.0.0.1:8000/case.html",+ 30.0,+ )+ self.assertEqual(+ fetch[:5],+ [+ "/bin/moli",+ "fetch",+ "--layout",+ "--resource",+ "http://127.0.0.1:8000/case.html",+ ],+ )+ def test_harness_timeout_multiplier_uses_wpt_case_metadata(self) -> None:+ self.assertEqual(_harness_timeout_multiplier(WptCase("normal.html")), 1.0)+ self.assertEqual(+ _harness_timeout_multiplier(+ WptCase("long.html", timeout_multiplier=LONG_TIMEOUT_MULTIPLIER)+ ),+ LONG_TIMEOUT_MULTIPLIER,+ )+ def test_lightpanda_cli_fetch_keeps_configured_timeout(self) -> None:+ command = _lightpanda_fetch(Path("/bin/lightpanda"), "http://127.0.0.1:8000/case.html", 30.0)++ self.assertEqual(command[:4], ["/bin/lightpanda", "fetch", "http://127.0.0.1:8000/case.html", "--dump"])+ self.assertEqual(command[command.index("--wait-until") + 1], "done")+ self.assertEqual(command[command.index("--wait-ms") + 1], "30000")+ self.assertEqual(command[command.index("--http-timeout") + 1], "30000")+ self.assertEqual(command[command.index("--terminate-ms") + 1], "30000")+ self.assertNotIn("--wait_until", command)+ self.assertNotIn("--wait_ms", command)+ self.assertNotIn("--http_timeout", command)+ def test_parser_exposes_configurable_parallelism_with_stable_timeout(self) -> None:+ parser = _build_parser()+ help_text = parser.format_help()++ self.assertEqual(WPT_CROSS_CASE_TIMEOUT_SECONDS, 120.0)+ self.assertEqual(WPT_CROSS_PARALLELISM, 50)+ self.assertNotIn("--case-timeout", help_text)+ self.assertNotIn("--case-timeout-engine", help_text)+ self.assertIn("--parallelism", help_text)+ self.assertNotIn("--cdp-parallelism", help_text)+ self.assertNotIn("--run-order", parser.format_help())+ self.assertNotIn("--shuffle-seed", parser.format_help())+ required = [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ ]+ self.assertEqual(parser.parse_args(required).parallelism, 50)+ self.assertEqual(+ parser.parse_args([*required, "--parallelism", "7"]).parallelism,+ 7,+ )+ def test_layout_profiles_are_explicit_and_keep_fixed_parallelism(self) -> None:+ parser = _build_parser()+ args = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--profile",+ "layout",+ ]+ )++ self.assertEqual(args.profile, "layout")+ self.assertEqual(+ LAYOUT_PROFILE_DIR_PREFIXES,+ (+ "css/css-flexbox",+ "css/css-grid",+ "css/css-sizing",+ "css/cssom-view",+ ),+ )+ self.assertEqual(+ (LAYOUT_VIEWPORT.width, LAYOUT_VIEWPORT.height),+ (800, 600),+ )+ self.assertEqual(LAYOUT_VIEWPORT.device_scale_factor, 1.0)+ self.assertEqual(WPT_CROSS_PARALLELISM, 50)+ def test_all_profile_matrix_deduplicates_default_and_layout_cases(self) -> None:+ semantic = WptCase("css/cssom-view/shared.html")+ duplicate_layout = WptCase("css/cssom-view/shared.html")+ reftest = WptCase(+ "css/css-grid/reference.html",+ test_type="reftest",+ references=(+ ReftestReference("css/css-grid/reference-ref.html", "=="),+ ),+ )++ merged = _deduplicate_cases([semantic, duplicate_layout, reftest])++ self.assertEqual(+ [case.case_path for case in merged],+ ["css/css-grid/reference.html", "css/cssom-view/shared.html"],+ )+ self.assertEqual(merged[0].test_type, "reftest")+ def test_manifest_reftest_enumeration_supports_relations_fuzzy_and_filters(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)++ def write_document(rel: str, body: str = "<!doctype html><p>static</p>") -> None:+ path = root / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(body, encoding="utf-8")++ documents = {+ "css/css-flexbox/static.html": "<!doctype html><meta name=timeout content=long><p>test</p>",+ "css/css-flexbox/ref.html": "<!doctype html><p>reference</p>",+ "css/css-flexbox/notref.html": "<!doctype html><p>not reference</p>",+ "css/css-flexbox/animation/dynamic.html": "<!doctype html><style>p { animation: pulse 1s }</style>",+ "css/css-flexbox/media.html": "<!doctype html><video></video>",+ "css/css-flexbox/server.html": "<!doctype html><script src='/handler.py'></script>",+ "css/css-flexbox/protocol.h2.html": "<!doctype html><p>h2</p>",+ "css/css-flexbox/driver.html": "<!doctype html><p>driver</p>",+ }+ for rel, body in documents.items():+ write_document(rel, body)++ reftests: dict[str, object] = {}++ def add_manifest_item(rel: str, item: list[object]) -> None:+ node = reftests+ parts = rel.split("/")+ for part in parts[:-1]:+ child = node.setdefault(part, {})+ assert isinstance(child, dict)+ node = child+ node[parts[-1]] = ["sha", item]++ add_manifest_item(+ "css/css-flexbox/static.html",+ [+ None,+ [+ ["/css/css-flexbox/ref.html", "=="],+ ["/css/css-flexbox/notref.html", "!="],+ ],+ {+ "timeout": "long",+ "fuzzy": [+ [None, [[0, 1], [0, 2]]],+ [+ [+ "/css/css-flexbox/static.html",+ "/css/css-flexbox/notref.html",+ "!=",+ ],+ [[0, 3], [0, 4]],+ ],+ ],+ },+ ],+ )+ for rel in (+ "css/css-flexbox/animation/dynamic.html",+ "css/css-flexbox/media.html",+ "css/css-flexbox/server.html",+ "css/css-flexbox/protocol.h2.html",+ ):+ add_manifest_item(+ rel,+ [None, [["/css/css-flexbox/ref.html", "=="]], {}],+ )+ add_manifest_item(+ "css/css-flexbox/driver.html",+ [+ None,+ [["/css/css-flexbox/ref.html", "=="]],+ {"testdriver": True},+ ],+ )+ (root / "MANIFEST.json").write_text(+ json.dumps(+ {+ "version": 9,+ "url_base": "/",+ "items": {"reftest": reftests},+ }+ ),+ encoding="utf-8",+ )++ cases = enumerate_reftest_cases(+ root,+ dir_prefixes=("css/css-flexbox",),+ )++ self.assertEqual([case.case_path for case in cases], ["css/css-flexbox/static.html"])+ case = cases[0]+ self.assertEqual(case.test_type, "reftest")+ self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ self.assertEqual(+ case.references,+ (+ ReftestReference(+ "css/css-flexbox/ref.html",+ "==",+ FuzzyTolerance((0, 1), (0, 2)),+ ),+ ReftestReference(+ "css/css-flexbox/notref.html",+ "!=",+ FuzzyTolerance((0, 3), (0, 4)),+ ),+ ),+ )+ self.assertEqual(+ explicit_reftest_case(root, "css/css-flexbox/static.html"),+ case,+ )+ def test_reftest_pixel_comparison_supports_exact_and_fuzzy_bounds(self) -> None:+ def captured(image: Image.Image) -> CapturedScreenshot:+ stream = io.BytesIO()+ image.save(stream, format="PNG")+ png = stream.getvalue()+ return CapturedScreenshot(+ png=png,+ sha256=hashlib.sha256(png).hexdigest(),+ width=image.width,+ height=image.height,+ )++ test_image = Image.new("RGB", (4, 4), (0, 0, 0))+ reference_image = test_image.copy()+ reference_image.putpixel((2, 1), (2, 0, 0))+ test_png = captured(test_image)+ reference_png = captured(reference_image)++ exact_equal, exact_metrics, exact_diff = compare_reftest_screenshots(+ test_png,+ reference_png,+ None,+ )+ fuzzy_equal, fuzzy_metrics, fuzzy_diff = compare_reftest_screenshots(+ test_png,+ reference_png,+ FuzzyTolerance((0, 2), (0, 1)),+ )+ too_strict, _, strict_diff = compare_reftest_screenshots(+ test_png,+ reference_png,+ FuzzyTolerance((0, 1), (0, 1)),+ )+ self.addCleanup(exact_diff.close)+ self.addCleanup(fuzzy_diff.close)+ self.addCleanup(strict_diff.close)++ self.assertFalse(exact_equal)+ self.assertEqual(exact_metrics["max_difference"], 2)+ self.assertEqual(exact_metrics["different_pixels"], 1)+ self.assertTrue(fuzzy_equal)+ self.assertEqual(+ fuzzy_metrics["fuzzy"],+ {"max_difference": [0, 2], "total_pixels": [0, 1]},+ )+ self.assertFalse(too_strict)+ def test_reftest_failure_artifacts_write_test_reference_and_diff_pngs(self) -> None:+ def captured(color: tuple[int, int, int]) -> CapturedScreenshot:+ image = Image.new("RGB", (3, 2), color)+ stream = io.BytesIO()+ image.save(stream, format="PNG")+ image.close()+ png = stream.getvalue()+ return CapturedScreenshot(+ png=png,+ sha256=hashlib.sha256(png).hexdigest(),+ width=3,+ height=2,+ )++ test_png = captured((255, 255, 255))+ reference_png = captured((0, 0, 0))+ diff_image = Image.new("RGB", (3, 2), (255, 255, 255))+ self.addCleanup(diff_image.close)+ evidence = _ReftestEvidence(+ reference=ReftestReferenceRun(+ reference_path="css/example-ref.html",+ url="http://example.test/css/example-ref.html",+ relation="==",+ ),+ screenshot=reference_png,+ diff_image=diff_image,+ )++ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir)+ artifacts = _write_reftest_failure_artifacts(+ output_dir=output_dir,+ engine="moli",+ case_path="css/example.html",+ test_screenshot=test_png,+ evidence=[evidence],+ )+ written = sorted(+ path.name+ for path in (output_dir / artifacts["directory"]).glob("*.png")+ )+ artifact_paths = [+ artifacts["test"],+ artifacts["references"][0]["reference"],+ artifacts["references"][0]["diff"],+ ]++ self.assertEqual(+ written,+ ["diff-01.png", "reference-01.png", "test.png"],+ )+ self.assertTrue(+ all((output_dir / artifact_path).stat().st_size > 0 for artifact_path in artifact_paths)+ )+ def test_reftest_match_and_mismatch_relationship_semantics(self) -> None:+ self.assertTrue(reftest_relation_passes("==", equal=True))+ self.assertFalse(reftest_relation_passes("==", equal=False))+ self.assertTrue(reftest_relation_passes("!=", equal=False))+ self.assertFalse(reftest_relation_passes("!=", equal=True))+ self.assertTrue(+ reftest_comparisons_pass(+ [+ {"relation": "==", "passed": False},+ {"relation": "==", "passed": True},+ {"relation": "!=", "passed": True},+ ]+ )+ )+ self.assertFalse(+ reftest_comparisons_pass(+ [+ {"relation": "==", "passed": True},+ {"relation": "!=", "passed": False},+ ]+ )+ )+ def test_fixed_run_schedule_is_deterministic(self) -> None:+ cases = [+ "html/browsers/a.html",+ "html/browsers/b.html",+ "content-security-policy/navigation/a.html",+ "content-security-policy/navigation/b.html",+ "trusted-types/reporting/a.html",+ "trusted-types/reporting/b.html",+ ]++ scheduled_a, metadata_a = build_run_schedule(+ cases,+ case_path=lambda case: case,+ )+ scheduled_b, metadata_b = build_run_schedule(+ cases,+ case_path=lambda case: case,+ )++ self.assertEqual(scheduled_a, scheduled_b)+ self.assertEqual(metadata_a, metadata_b)+ self.assertCountEqual(scheduled_a, cases)+ self.assertNotEqual(scheduled_a, cases)+ self.assertEqual(metadata_a["mode"], "fixed-prefix-balanced-shuffle")+ self.assertEqual(metadata_a["seed"], FIXED_RUN_SHUFFLE_SEED)+ buckets = [case.split("/", 1)[0] for case in scheduled_a]+ self.assertTrue(all(left != right for left, right in zip(buckets, buckets[1:])))diff --git a/moli-benchmark/tests/test_wpt_cross_case_enumeration.py b/moli-benchmark/tests/test_wpt_cross_case_enumeration.pynew file mode 100644index 000000000..be27f80c5--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_case_enumeration.py@@ -0,0 +1,1189 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossCaseEnumerationTests(WptCrossTestCase):+ def test_enumerate_cases_includes_htm_before_applying_limit(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "dom"+ case_dir.mkdir()+ harness = '<script src="/resources/testharness.js"></script>'+ (case_dir / "a.htm").write_text(+ '<meta name="timeout" content="long">'+ '<meta name="variant" content="?mode=z">'+ '<meta name="variant" content="?mode=a">'+ + harness,+ encoding="utf-8",+ )+ (case_dir / "z.html").write_text(harness, encoding="utf-8")+ expected = [+ WptCase("dom/a.htm?mode=a", LONG_TIMEOUT_MULTIPLIER),+ WptCase("dom/a.htm?mode=z", LONG_TIMEOUT_MULTIPLIER),+ WptCase("dom/z.html"),+ ]++ for prefixes in (None, ("dom",)):+ with self.subTest(dir_prefixes=prefixes):+ self.assertEqual(+ enumerate_cases(wpt_root, dir_prefixes=prefixes), expected+ )+ self.assertEqual(+ enumerate_cases(wpt_root, dir_prefixes=prefixes, limit=1),+ expected[:1],+ )+ def test_enumerate_cases_expands_wpt_meta_variants(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "dom" / "ranges"+ case_dir.mkdir(parents=True)+ (case_dir / "variant.html").write_text(+ """<!doctype html>+<meta name="variant" content="?mode=open">+<meta name="variant" content="?mode=closed#frag">+<meta name="variant" content="?command=heading¶m=h1">+<meta name="variant" content="?mode=encoded&flag=1">+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (case_dir / "plain.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (case_dir / "long.html").write_text(+ """<!doctype html>+<meta name="timeout" content="long">+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("dom/ranges",))++ self.assertEqual(+ [case.case_path for case in cases],+ [+ "dom/ranges/long.html",+ "dom/ranges/plain.html",+ "dom/ranges/variant.html?command=heading¶m=h1",+ "dom/ranges/variant.html?mode=closed#frag",+ "dom/ranges/variant.html?mode=encoded&flag=1",+ "dom/ranges/variant.html?mode=open",+ ],+ )+ timeouts = {case.case_path: case.timeout_multiplier for case in cases}+ self.assertEqual(timeouts["dom/ranges/long.html"], LONG_TIMEOUT_MULTIPLIER)+ self.assertEqual(timeouts["dom/ranges/plain.html"], 1.0)+ self.assertEqual(timeouts["dom/ranges/variant.html?mode=open"], 1.0)+ def test_enumerate_cases_skips_wptserve_python_resources(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "html" / "semantics"+ common_dir = wpt_root / "common" / "security-features" / "resources"+ case_dir.mkdir(parents=True)+ common_dir.mkdir(parents=True)+ (case_dir / "direct.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>fetch("/resources/inspect.py?cmd=get");</script>+""",+ encoding="utf-8",+ )+ (case_dir / "external.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="helper.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (case_dir / "helper.js").write_text(+ 'fetch("/resources/inspect-header.py?cmd=get");',+ encoding="utf-8",+ )+ (case_dir / "kept.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (case_dir / "generated-comment.html").write_text(+ """<!doctype html>+<!-- DO NOT EDIT! This test has been generated by /html/canvas/tools/gentest.py. -->+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (case_dir / "absolute-helper.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/common/security-features/resources/common.sub.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (common_dir / "common.sub.js").write_text(+ 'fetch("/common/security-features/subresource/xhr.py");',+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("html/semantics",))++ self.assertEqual(+ [case.case_path for case in cases],+ [+ "html/semantics/generated-comment.html",+ "html/semantics/kept.html",+ ],+ )+ def test_default_enumeration_uses_non_layout_blacklist_without_encoding(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ dom_dir = wpt_root / "dom" / "nodes"+ encoding_dir = wpt_root / "encoding" / "legacy-mb-japanese"+ canvas_dir = wpt_root / "html" / "canvas" / "element"+ layout_dir = wpt_root / "css" / "css-grid"+ container_query_dir = (+ wpt_root / "css" / "css-conditional" / "container-queries"+ )+ css_conditional_dir = wpt_root / "css" / "css-conditional"+ css_animation_dir = wpt_root / "css" / "css-animations" / "parsing"+ css_animation_timeline_dir = wpt_root / "css" / "css-animations"+ css_transition_dir = wpt_root / "css" / "css-transitions"+ css_transition_parsing_dir = (+ wpt_root / "css" / "css-transitions" / "parsing"+ )+ css_paint_dir = wpt_root / "css" / "css-paint-api"+ css_scrollbar_dir = wpt_root / "css" / "css-scrollbars"+ css_viewport_dir = wpt_root / "css" / "css-viewport" / "zoom"+ css_viewport_parsing_dir = (+ wpt_root / "css" / "css-viewport" / "zoom" / "parsing"+ )+ typed_om_dir = wpt_root / "css" / "css-typed-om"+ render_blocking_dir = wpt_root / "html" / "dom" / "render-blocking"+ service_worker_dir = wpt_root / "service-workers" / "service-worker"+ service_worker_named_dir = wpt_root / "webusb"+ non_service_worker_https_dir = wpt_root / "WebCryptoAPI"+ media_dir = (+ wpt_root / "html" / "semantics" / "embedded-content" / "media-elements"+ )+ iframe_dir = (+ wpt_root+ / "html"+ / "semantics"+ / "embedded-content"+ / "the-iframe-element"+ )+ dom_dir.mkdir(parents=True)+ encoding_dir.mkdir(parents=True)+ canvas_dir.mkdir(parents=True)+ layout_dir.mkdir(parents=True)+ container_query_dir.mkdir(parents=True)+ css_conditional_dir.mkdir(parents=True, exist_ok=True)+ css_animation_dir.mkdir(parents=True)+ css_transition_dir.mkdir(parents=True)+ css_transition_parsing_dir.mkdir(parents=True)+ css_paint_dir.mkdir(parents=True)+ css_scrollbar_dir.mkdir(parents=True)+ css_viewport_dir.mkdir(parents=True)+ css_viewport_parsing_dir.mkdir(parents=True)+ typed_om_dir.mkdir(parents=True)+ render_blocking_dir.mkdir(parents=True)+ service_worker_dir.mkdir(parents=True)+ service_worker_named_dir.mkdir(parents=True)+ non_service_worker_https_dir.mkdir(parents=True)+ media_dir.mkdir(parents=True)+ iframe_dir.mkdir(parents=True)+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+"""+ (dom_dir / "kept.html").write_text(case_html, encoding="utf-8")+ (encoding_dir / "kept.html").write_text(case_html, encoding="utf-8")+ (canvas_dir / "filtered.html").write_text(case_html, encoding="utf-8")+ (layout_dir / "filtered.html").write_text(case_html, encoding="utf-8")+ (container_query_dir / "at-container-parsing.html").write_text(+ case_html, encoding="utf-8"+ )+ (container_query_dir / "size-feature-evaluation.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_conditional_dir / "at-supports-named-feature-001.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_animation_dir / "kept.html").write_text(case_html, encoding="utf-8")+ (css_animation_timeline_dir / "animate-with-color-mix.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_transition_dir / "events-001.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_transition_parsing_dir / "transition-computed.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_paint_dir / "filtered.html").write_text(case_html, encoding="utf-8")+ (css_scrollbar_dir / "scrollbar-width-001.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_scrollbar_dir / "scrollbar-width-parsing.html").write_text(+ case_html, encoding="utf-8"+ )+ (css_viewport_dir / "widget.html").write_text(case_html, encoding="utf-8")+ (css_viewport_parsing_dir / "zoom-computed.html").write_text(+ case_html, encoding="utf-8"+ )+ (typed_om_dir / "kept.html").write_text(case_html, encoding="utf-8")+ (render_blocking_dir / "filtered.html").write_text(+ case_html, encoding="utf-8"+ )+ (service_worker_dir / "registration.https.html").write_text(+ case_html, encoding="utf-8"+ )+ (service_worker_named_dir / "usb.serviceworker.https.html").write_text(+ case_html, encoding="utf-8"+ )+ (+ non_service_worker_https_dir+ / "crypto-subtle-secure-context-available.https.sub.html"+ ).write_text(case_html, encoding="utf-8")+ (media_dir / "filtered.html").write_text(case_html, encoding="utf-8")+ (iframe_dir / "kept.html").write_text(case_html, encoding="utf-8")++ cases = enumerate_cases(wpt_root)+ focused_transition_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("css/css-transitions",),+ )++ self.assertNotIn("encoding", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertIn("html/canvas", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertIn("css/css-grid", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertNotIn(+ "css/css-conditional/container-queries", DEFAULT_EXCLUDE_DIR_PREFIXES+ )+ self.assertIn("css/css-paint-api", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertNotIn("css/css-animations", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertNotIn("css/css-transitions", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertIn("html/dom/render-blocking", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertNotIn("service-workers", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertIn(+ "html/semantics/embedded-content/media-elements",+ DEFAULT_EXCLUDE_DIR_PREFIXES,+ )+ self.assertNotIn("css/css-typed-om", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertNotIn(+ "html/semantics/embedded-content/the-iframe-element",+ DEFAULT_EXCLUDE_DIR_PREFIXES,+ )+ self.assertEqual(+ [case.case_path for case in cases],+ [+ "css/css-animations/parsing/kept.html",+ "css/css-conditional/container-queries/at-container-parsing.html",+ "css/css-scrollbars/scrollbar-width-parsing.html",+ "css/css-transitions/parsing/transition-computed.html",+ "css/css-typed-om/kept.html",+ "css/css-viewport/zoom/parsing/zoom-computed.html",+ "dom/nodes/kept.html",+ "encoding/legacy-mb-japanese/kept.html",+ "html/semantics/embedded-content/the-iframe-element/kept.html",+ "service-workers/service-worker/registration.https.html",+ "webusb/usb.serviceworker.https.html",+ ],+ )+ self.assertEqual(+ [case.case_path for case in focused_transition_cases],+ [+ "css/css-transitions/events-001.html",+ "css/css-transitions/parsing/transition-computed.html",+ ],+ )+ def test_default_enumeration_excludes_jpegxl_codec_fidelity(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ jpegxl_dir = wpt_root / "jpegxl"+ jpegxl_dir.mkdir(parents=True)+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+"""+ (jpegxl_dir / "decode.html").write_text(case_html, encoding="utf-8")++ default_cases = enumerate_cases(wpt_root)+ focused_cases = enumerate_cases(wpt_root, dir_prefixes=("jpegxl",))++ self.assertIn("jpegxl", DEFAULT_EXCLUDE_DIR_PREFIXES)+ self.assertEqual(default_cases, [])+ self.assertEqual(+ [case.case_path for case in focused_cases],+ ["jpegxl/decode.html"],+ )+ def test_enumerate_cases_skips_manual_and_support_resource_pages(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "FileAPI"+ resource_dir = wpt_root / "appmanifest" / "resources"+ support_dir = wpt_root / "workers" / "support"+ case_dir.mkdir(parents=True)+ resource_dir.mkdir(parents=True)+ support_dir.mkdir(parents=True)+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+"""+ for suffix in (".html", ".htm"):+ (case_dir / f"kept{suffix}").write_text(case_html, encoding="utf-8")+ (case_dir / f"upload-manual{suffix}").write_text(+ case_html, encoding="utf-8"+ )+ (resource_dir / f"helper{suffix}").write_text(+ case_html, encoding="utf-8"+ )+ (support_dir / f"helper{suffix}").write_text(+ case_html, encoding="utf-8"+ )++ cases = enumerate_cases(wpt_root)++ self.assertEqual(+ [case.case_path for case in cases],+ ["FileAPI/kept.htm", "FileAPI/kept.html"],+ )+ def test_explicit_dir_prefix_bypasses_default_rendering_blacklist(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ canvas_dir = wpt_root / "html" / "canvas"+ canvas_dir.mkdir(parents=True)+ (canvas_dir / "case.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("html/canvas",))++ self.assertEqual([case.case_path for case in cases], ["html/canvas/case.html"])+ def test_preload_dir_prefix_allows_modulepreload_helper_without_stash_usage(+ self,+ ) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ preload_dir = wpt_root / "preload"+ resources_dir = preload_dir / "resources"+ resources_dir.mkdir(parents=True)+ harness_dir = wpt_root / "resources"+ harness_dir.mkdir(parents=True)+ (harness_dir / "testharness.js").write_text("", encoding="utf-8")+ (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")+ (resources_dir / "preload_helper.js").write_text(+ "function unused() { return '/preload/resources/stash-put.py'; }",+ encoding="utf-8",+ )+ modulepreload_case = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script src="/preload/resources/preload_helper.js"></script>+<link rel=modulepreload href="resources/dummy.js?pipe=trickle(d5)">+<script>promise_test(async () => {}, "ok");</script>+"""+ (preload_dir / "avoid-delaying-onload-link-modulepreload.html").write_text(+ modulepreload_case,+ encoding="utf-8",+ )+ (+ preload_dir / "avoid-delaying-onload-link-modulepreload-exec.html"+ ).write_text(+ modulepreload_case,+ encoding="utf-8",+ )+ (preload_dir / "avoid-delaying-onload-link-preload.html").write_text(+ modulepreload_case,+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("preload",))++ self.assertEqual(+ [case.case_path for case in cases],+ [+ "preload/avoid-delaying-onload-link-modulepreload-exec.html",+ "preload/avoid-delaying-onload-link-modulepreload.html",+ ],+ )+ def test_preload_dir_prefix_excludes_sri_non_goal_cases(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ preload_dir = wpt_root / "preload"+ preload_dir.mkdir(parents=True)+ harness_dir = wpt_root / "resources"+ harness_dir.mkdir(parents=True)+ (harness_dir / "testharness.js").write_text("", encoding="utf-8")+ (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>promise_test(async () => {}, "ok");</script>+"""+ (preload_dir / "modulepreload-json.html").write_text(+ case_html,+ encoding="utf-8",+ )+ (preload_dir / "modulepreload-sri.html").write_text(+ case_html,+ encoding="utf-8",+ )+ (preload_dir / "modulepreload-sri-importmap.html").write_text(+ case_html,+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("preload",))++ self.assertEqual(+ [case.case_path for case in cases],+ ["preload/modulepreload-json.html"],+ )+ def test_enumerate_cases_excludes_malformed_select_fragment_non_goal(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ parsing_dir = wpt_root / "html" / "syntax" / "parsing"+ parsing_dir.mkdir(parents=True)+ harness_dir = wpt_root / "resources"+ harness_dir.mkdir(parents=True)+ (harness_dir / "testharness.js").write_text("", encoding="utf-8")+ (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>test(() => {}, "ok");</script>+"""+ (parsing_dir / "html5lib_innerHTML_webkit02.html").write_text(+ case_html,+ encoding="utf-8",+ )+ (parsing_dir / "html5lib_scripted_webkit01.html").write_text(+ case_html,+ encoding="utf-8",+ )++ default_cases = enumerate_cases(wpt_root)+ focused_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("html/syntax/parsing",),+ )++ expected = ["html/syntax/parsing/html5lib_scripted_webkit01.html"]+ self.assertEqual([case.case_path for case in default_cases], expected)+ self.assertEqual([case.case_path for case in focused_cases], expected)+ def test_enumerate_cases_excludes_formdata_non_goal_cases(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ formdata_dir = wpt_root / "xhr" / "formdata"+ formdata_dir.mkdir(parents=True)+ harness_dir = wpt_root / "resources"+ harness_dir.mkdir(parents=True)+ (harness_dir / "testharness.js").write_text("", encoding="utf-8")+ (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>test(() => {}, "ok");</script>+"""+ (formdata_dir / "constructor-formelement.html").write_text(+ case_html,+ encoding="utf-8",+ )+ (formdata_dir / "constructor-submitter-coordinate.html").write_text(+ case_html,+ encoding="utf-8",+ )+ (formdata_dir / "constructor-submitter.html").write_text(+ case_html,+ encoding="utf-8",+ )++ default_cases = enumerate_cases(wpt_root)+ focused_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("xhr/formdata",),+ )+ diagnostic_case = explicit_case(+ wpt_root,+ "xhr/formdata/constructor-formelement.html",+ )+ coordinate_diagnostic_case = explicit_case(+ wpt_root,+ "xhr/formdata/constructor-submitter-coordinate.html",+ )++ expected = ["xhr/formdata/constructor-submitter.html"]+ self.assertEqual([case.case_path for case in default_cases], expected)+ self.assertEqual([case.case_path for case in focused_cases], expected)+ self.assertEqual(+ diagnostic_case.case_path,+ "xhr/formdata/constructor-formelement.html",+ )+ self.assertEqual(+ coordinate_diagnostic_case.case_path,+ "xhr/formdata/constructor-submitter-coordinate.html",+ )+ def test_enumerate_cases_allows_supported_xhr_delay_handler(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ xhr_dir = wpt_root / "xhr"+ xhr_dir.mkdir(parents=True)+ harness_dir = wpt_root / "resources"+ harness_dir.mkdir(parents=True)+ (harness_dir / "testharness.js").write_text("", encoding="utf-8")+ (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")+ (xhr_dir / "delay-supported.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>fetch("resources/delay.py?ms=1"); test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )+ (xhr_dir / "delay-and-unsupported.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>fetch("resources/delay.py"); fetch("resources/other.py");</script>+""",+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("xhr",))++ self.assertEqual(+ [case.case_path for case in cases],+ ["xhr/delay-supported.html"],+ )+ def test_enumerate_cases_allows_supported_delayed_module_handler(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ harness_dir = wpt_root / "resources"+ harness_dir.mkdir(parents=True)+ (harness_dir / "testharness.js").write_text("", encoding="utf-8")+ (harness_dir / "testharnessreport.js").write_text("", encoding="utf-8")+ module_dir = (+ wpt_root+ / "html"+ / "semantics"+ / "scripting-1"+ / "the-script-element"+ / "module"+ )+ (module_dir / "resources").mkdir(parents=True)+ (module_dir / "delay-supported.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script type="module" src="./delay-supported.js"></script>+""",+ encoding="utf-8",+ )+ (module_dir / "delay-supported.js").write_text(+ 'import "./resources/delayed-modulescript.py?ms=1";',+ encoding="utf-8",+ )+ (module_dir / "resources" / "delayed-modulescript.py").write_text(+ "# modeled by wpt-cross",+ encoding="utf-8",+ )++ cases = enumerate_cases(+ wpt_root,+ dir_prefixes=(+ "html/semantics/scripting-1/the-script-element/module",+ ),+ )++ self.assertEqual(+ [case.case_path for case in cases],+ [+ "html/semantics/scripting-1/the-script-element/module/"+ "delay-supported.html"+ ],+ )+ def test_enumerate_cases_wraps_window_js_for_explicit_dir_prefix(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ api_dir = wpt_root / "WebCryptoAPI"+ api_dir.mkdir(parents=True)+ (api_dir / "algorithm-discards-context.https.window.js").write_text(+ """// META: title=window case+// META: timeout=long+promise_test(async () => {}, "ok");+""",+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root, dir_prefixes=("WebCryptoAPI",))++ self.assertEqual(+ [case.case_path for case in cases],+ [+ "WebCryptoAPI/algorithm-discards-context.https.window.js?moli-wpt-script=window",+ ],+ )+ self.assertEqual(cases[0].timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ def test_explicit_dir_prefix_includes_simple_https_sub_html(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ api_dir = wpt_root / "WebCryptoAPI" / "secure_context"+ api_dir.mkdir(parents=True)+ (api_dir / "crypto-subtle-secure-context-available.https.sub.html").write_text(+ """<!doctype html>+<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>test(() => assert_equals("{{host}}", "{{host}}"), "ok");</script>+""",+ encoding="utf-8",+ )++ default_cases = enumerate_cases(wpt_root)+ selected_cases = enumerate_cases(wpt_root, dir_prefixes=("WebCryptoAPI",))++ self.assertEqual(default_cases, [])+ self.assertEqual(+ [case.case_path for case in selected_cases],+ ["WebCryptoAPI/secure_context/crypto-subtle-secure-context-available.https.sub.html"],+ )+ def test_dir_prefix_can_opt_into_tentative_cases(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "wasm" / "webapi" / "esm-integration"+ case_dir.mkdir(parents=True)+ case_html = """<!doctype html>+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+"""+ (case_dir / "stable.html").write_text(case_html, encoding="utf-8")+ (case_dir / "feature.tentative.html").write_text(+ case_html,+ encoding="utf-8",+ )++ default_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/webapi/esm-integration",),+ )+ tentative_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/webapi/esm-integration",),+ include_tentative=True,+ )++ self.assertEqual(+ [case.case_path for case in default_cases],+ ["wasm/webapi/esm-integration/stable.html"],+ )+ self.assertEqual(+ [case.case_path for case in tentative_cases],+ [+ "wasm/webapi/esm-integration/feature.tentative.html",+ "wasm/webapi/esm-integration/stable.html",+ ],+ )+ def test_dir_prefix_can_opt_into_any_js_globals(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "wasm" / "jsapi" / "js-string"+ resource_dir = case_dir / "resources"+ case_dir.mkdir(parents=True)+ resource_dir.mkdir(parents=True)+ (case_dir / "basic.any.js").write_text(+ """// META: timeout=long+promise_test(async () => {}, "ok");+""",+ encoding="utf-8",+ )+ (case_dir / "feature.tentative.any.js").write_text(+ """promise_test(async () => {}, "ok");""",+ encoding="utf-8",+ )+ (case_dir / "testdriver.any.js").write_text(+ """// META: script=/resources/testdriver.js+promise_test(async () => {}, "requires testdriver");+""",+ encoding="utf-8",+ )+ (resource_dir / "helper.any.js").write_text(+ """promise_test(async () => {}, "not a top-level case");""",+ encoding="utf-8",+ )++ default_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/jsapi/js-string",),+ )+ window_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/jsapi/js-string",),+ any_js_global=ANY_JS_WINDOW_GLOBAL,+ )+ both_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/jsapi/js-string",),+ include_tentative=True,+ any_js_global="both",+ )++ self.assertEqual(default_cases, [])+ self.assertEqual(+ [(case.case_path, case.timeout_multiplier) for case in window_cases],+ [+ (+ "wasm/jsapi/js-string/basic.any.js?moli-wpt-any=window",+ LONG_TIMEOUT_MULTIPLIER,+ )+ ],+ )+ self.assertEqual(+ [case.case_path for case in both_cases],+ [+ "wasm/jsapi/js-string/basic.any.js?moli-wpt-any=dedicatedworker",+ "wasm/jsapi/js-string/basic.any.js?moli-wpt-any=window",+ "wasm/jsapi/js-string/feature.tentative.any.js?moli-wpt-any=dedicatedworker",+ "wasm/jsapi/js-string/feature.tentative.any.js?moli-wpt-any=window",+ ],+ )+ def test_default_enumeration_includes_streams_script_cases(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ streams_dir = wpt_root / "streams" / "readable-streams"+ other_dir = wpt_root / "wasm" / "jsapi"+ streams_dir.mkdir(parents=True)+ other_dir.mkdir(parents=True)+ (streams_dir / "general.any.js").write_text(+ """// META: global=window,worker+promise_test(async () => {}, "general");+""",+ encoding="utf-8",+ )+ (streams_dir / "window-only.any.js").write_text(+ """// META: global=window+test(() => {}, "window only");+""",+ encoding="utf-8",+ )+ (streams_dir / "task.window.js").write_text(+ """// META: timeout=long+test(() => {}, "window script");+""",+ encoding="utf-8",+ )+ (streams_dir / "task.worker.js").write_text(+ """test(() => {}, "worker script");+done();+""",+ encoding="utf-8",+ )+ (streams_dir / "feature.tentative.any.js").write_text(+ """test(() => {}, "tentative");""",+ encoding="utf-8",+ )+ (other_dir / "excluded.any.js").write_text(+ """test(() => {}, "not in the broad baseline");""",+ encoding="utf-8",+ )++ cases = enumerate_cases(wpt_root)++ self.assertEqual(+ [(case.case_path, case.timeout_multiplier) for case in cases],+ [+ (+ "streams/readable-streams/general.any.js?moli-wpt-any=dedicatedworker",+ 1.0,+ ),+ (+ "streams/readable-streams/general.any.js?moli-wpt-any=window",+ 1.0,+ ),+ (+ "streams/readable-streams/task.window.js?moli-wpt-script=window",+ LONG_TIMEOUT_MULTIPLIER,+ ),+ (+ "streams/readable-streams/task.worker.js?moli-wpt-script=dedicatedworker",+ 1.0,+ ),+ (+ "streams/readable-streams/window-only.any.js?moli-wpt-any=window",+ 1.0,+ ),+ ],+ )+ def test_default_enumeration_includes_compression_script_cases(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ sources = {+ "compression/compression-stream.any.js": "// META: global=window,worker\n",+ "compression/idlharness.https.any.js": "// META: global=window,dedicatedworker\n",+ "compression/compression-with-detach.window.js": "",+ "compression/decompression-with-detach.window.js": "",+ "compression/feature.tentative.any.js": "",+ "compression/testdriver.any.js": "// META: script=/resources/testdriver.js\n",+ "compression/resources/helper.any.js": "",+ "compression-other/excluded.any.js": "",+ "compression-other/excluded.window.js": "",+ }+ for name, meta in sources.items():+ path = wpt_root / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(meta + 'test(() => {}, "case");', encoding="utf-8")++ cases = enumerate_cases(wpt_root)+ focused_cases = enumerate_cases(+ wpt_root, dir_prefixes=("compression",), any_js_global="both"+ )+ window_cases = enumerate_cases(wpt_root, any_js_global="window")++ expected = [+ "compression/compression-stream.any.js?moli-wpt-any=dedicatedworker",+ "compression/compression-stream.any.js?moli-wpt-any=window",+ "compression/compression-with-detach.window.js?moli-wpt-script=window",+ "compression/decompression-with-detach.window.js?moli-wpt-script=window",+ "compression/idlharness.https.any.js?moli-wpt-any=dedicatedworker",+ "compression/idlharness.https.any.js?moli-wpt-any=window",+ ]+ self.assertEqual([case.case_path for case in cases], expected)+ self.assertEqual([case.case_path for case in focused_cases], expected)+ self.assertEqual(+ [case.case_path for case in window_cases if case.case_path.startswith("compression/")],+ [path for path in expected if "dedicatedworker" not in path],+ )+ def test_dir_prefix_expands_any_js_variants_and_respects_globals(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "WebCryptoAPI"+ case_dir.mkdir(parents=True)+ (case_dir / "multi.any.js").write_text(+ """// META: global=window,worker+// META: timeout=long+// META: variant=?mode=1+// META: variant=?mode=2+promise_test(async () => {}, "multi");+""",+ encoding="utf-8",+ )+ (case_dir / "window-only.any.js").write_text(+ """// META: global=window+promise_test(async () => {}, "window");+""",+ encoding="utf-8",+ )+ (case_dir / "worker-only.any.js").write_text(+ """// META: global=dedicatedworker+promise_test(async () => {}, "worker");+""",+ encoding="utf-8",+ )++ cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("WebCryptoAPI",),+ any_js_global="both",+ )++ self.assertEqual(+ [(case.case_path, case.timeout_multiplier) for case in cases],+ [+ (+ "WebCryptoAPI/multi.any.js?mode=1&moli-wpt-any=dedicatedworker",+ LONG_TIMEOUT_MULTIPLIER,+ ),+ (+ "WebCryptoAPI/multi.any.js?mode=1&moli-wpt-any=window",+ LONG_TIMEOUT_MULTIPLIER,+ ),+ (+ "WebCryptoAPI/multi.any.js?mode=2&moli-wpt-any=dedicatedworker",+ LONG_TIMEOUT_MULTIPLIER,+ ),+ (+ "WebCryptoAPI/multi.any.js?mode=2&moli-wpt-any=window",+ LONG_TIMEOUT_MULTIPLIER,+ ),+ (+ "WebCryptoAPI/window-only.any.js?moli-wpt-any=window",+ 1.0,+ ),+ (+ "WebCryptoAPI/worker-only.any.js?moli-wpt-any=dedicatedworker",+ 1.0,+ ),+ ],+ )+ def test_dir_prefix_includes_window_and_worker_js_wrappers(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "wasm" / "serialization"+ case_dir.mkdir(parents=True)+ (case_dir / "transfer.window.js").write_text(+ """// META: timeout=long+test(() => {}, "window");+""",+ encoding="utf-8",+ )+ (case_dir / "memory.worker.js").write_text(+ """importScripts("/resources/testharness.js");+test(() => {}, "worker");+done();+""",+ encoding="utf-8",+ )+ (case_dir / "feature.tentative.window.js").write_text(+ """test(() => {}, "tentative");""",+ encoding="utf-8",+ )++ default_cases = enumerate_cases(wpt_root)+ focused_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/serialization",),+ )+ tentative_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm/serialization",),+ include_tentative=True,+ )++ self.assertEqual(default_cases, [])+ self.assertEqual(+ [(case.case_path, case.timeout_multiplier) for case in focused_cases],+ [+ (+ "wasm/serialization/memory.worker.js?moli-wpt-script=dedicatedworker",+ 1.0,+ ),+ (+ "wasm/serialization/transfer.window.js?moli-wpt-script=window",+ LONG_TIMEOUT_MULTIPLIER,+ ),+ ],+ )+ self.assertEqual(+ [case.case_path for case in tentative_cases],+ [+ "wasm/serialization/feature.tentative.window.js?moli-wpt-script=window",+ "wasm/serialization/memory.worker.js?moli-wpt-script=dedicatedworker",+ "wasm/serialization/transfer.window.js?moli-wpt-script=window",+ ],+ )+ def test_dir_prefix_includes_sub_https_and_supported_wasm_status_handler(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ serialization_dir = wpt_root / "wasm" / "serialization" / "module"+ serialization_dir.mkdir(parents=True)+ webapi_dir = wpt_root / "wasm" / "webapi"+ webapi_dir.mkdir(parents=True)+ esm_dir = webapi_dir / "esm-integration"+ esm_dir.mkdir()+ (serialization_dir / "window-domain-success.sub.html").write_text(+ """<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>test(() => {}, "sub");</script>+""",+ encoding="utf-8",+ )+ (esm_dir / "worklet-import-source-phase.tentative.https.html").write_text(+ """<script src="/resources/testharness.js"></script>+<script src="/resources/testharnessreport.js"></script>+<script>test(() => {}, "https tentative");</script>+""",+ encoding="utf-8",+ )+ (webapi_dir / "status.any.js").write_text(+ """// META: global=window,worker+promise_test(async t => {+ await promise_rejects_js(t, TypeError, WebAssembly.compileStreaming(fetch("status.py?status=404")));+}, "status");+""",+ encoding="utf-8",+ )+ (webapi_dir / "badstatus.any.js").write_text(+ """// META: global=window,worker+promise_test(async t => {+ await promise_rejects_js(t, TypeError, WebAssembly.compileStreaming(fetch("badstatus.py?status=404")));+}, "badstatus");+""",+ encoding="utf-8",+ )+ (webapi_dir / "origin.sub.any.js").write_text(+ """// META: global=window,worker+promise_test(async t => {+ await promise_rejects_js(t, TypeError, WebAssembly.compileStreaming(fetch("/fetch/api/resources/redirect.py?redirect_status=301&location=/wasm/incrementer.wasm")));+}, "origin");+""",+ encoding="utf-8",+ )++ broad_cases = enumerate_cases(wpt_root)+ focused_cases = enumerate_cases(+ wpt_root,+ dir_prefixes=("wasm",),+ include_tentative=True,+ any_js_global="both",+ )++ self.assertEqual(broad_cases, [])+ self.assertEqual(+ [case.case_path for case in focused_cases],+ [+ "wasm/serialization/module/window-domain-success.sub.html",+ "wasm/webapi/esm-integration/worklet-import-source-phase.tentative.https.html",+ "wasm/webapi/origin.sub.any.js?moli-wpt-any=dedicatedworker",+ "wasm/webapi/origin.sub.any.js?moli-wpt-any=window",+ "wasm/webapi/status.any.js?moli-wpt-any=dedicatedworker",+ "wasm/webapi/status.any.js?moli-wpt-any=window",+ ],+ )+ def test_explicit_case_bypasses_curated_filename_filters(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "shadow-dom"+ case_dir.mkdir(parents=True)+ (case_dir / "feature.tentative.html").write_text(+ """<!doctype html>+<meta name="timeout" content="long">+<script src="/resources/testharness.js"></script>+<script>test(() => {}, "ok");</script>+""",+ encoding="utf-8",+ )++ case = explicit_case(wpt_root, "shadow-dom/feature.tentative.html?variant=1")++ self.assertEqual(case.case_path, "shadow-dom/feature.tentative.html?variant=1")+ self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ def test_explicit_case_wraps_any_js(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "WebCryptoAPI"+ case_dir.mkdir(parents=True)+ (case_dir / "historical.any.js").write_text(+ """// META: global=window,dedicatedworker+// META: timeout=long+test(() => {}, "ok");+""",+ encoding="utf-8",+ )++ case = explicit_case(wpt_root, "WebCryptoAPI/historical.any.js?mode=1")++ self.assertEqual(+ case.case_path,+ "WebCryptoAPI/historical.any.js?mode=1&moli-wpt-any=window",+ )+ self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ def test_explicit_window_and_worker_js_cases_use_wrappers(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "wasm"+ case_dir.mkdir(parents=True)+ (case_dir / "transfer.window.js").write_text(+ """// META: timeout=long+test(() => {}, "ok");+""",+ encoding="utf-8",+ )+ (case_dir / "memory.worker.js").write_text(+ """test(() => {}, "ok");""",+ encoding="utf-8",+ )++ window_case = explicit_case(wpt_root, "wasm/transfer.window.js?variant=1")+ worker_case = explicit_case(wpt_root, "wasm/memory.worker.js")++ self.assertEqual(+ window_case.case_path,+ "wasm/transfer.window.js?variant=1&moli-wpt-script=window",+ )+ self.assertEqual(window_case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ self.assertEqual(+ worker_case.case_path,+ "wasm/memory.worker.js?moli-wpt-script=dedicatedworker",+ )+ def test_explicit_case_accepts_generated_any_html_path(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "WebCryptoAPI"+ case_dir.mkdir(parents=True)+ (case_dir / "historical.any.js").write_text(+ """// META: timeout=long+test(() => {}, "ok");+""",+ encoding="utf-8",+ )++ case = explicit_case(+ wpt_root,+ f"WebCryptoAPI/historical.any.html?{ANY_JS_WINDOW_QUERY}",+ )++ self.assertEqual(+ case.case_path,+ f"WebCryptoAPI/historical.any.html?{ANY_JS_WINDOW_QUERY}",+ )+ self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ def test_explicit_case_accepts_generated_window_html_path(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ wpt_root = Path(temp_dir)+ case_dir = wpt_root / "WebCryptoAPI"+ case_dir.mkdir(parents=True)+ (case_dir / "algorithm-discards-context.https.window.js").write_text(+ """// META: timeout=long+test(() => {}, "ok");+""",+ encoding="utf-8",+ )++ case = explicit_case(+ wpt_root,+ f"WebCryptoAPI/algorithm-discards-context.https.window.html?{WINDOW_JS_WINDOW_QUERY}",+ )++ self.assertEqual(+ case.case_path,+ f"WebCryptoAPI/algorithm-discards-context.https.window.html?{WINDOW_JS_WINDOW_QUERY}",+ )+ self.assertEqual(case.timeout_multiplier, LONG_TIMEOUT_MULTIPLIER)+ def test_any_js_window_case_path_does_not_duplicate_window_query(self) -> None:+ self.assertEqual(+ any_js_window_case_path(+ f"WebCryptoAPI/historical.any.js?{ANY_JS_WINDOW_QUERY}#frag"+ ),+ f"WebCryptoAPI/historical.any.html?{ANY_JS_WINDOW_QUERY}#frag",+ )+ def test_window_js_window_case_path_does_not_duplicate_window_query(self) -> None:+ self.assertEqual(+ window_js_window_case_path(+ f"WebCryptoAPI/example.window.js?{WINDOW_JS_WINDOW_QUERY}#frag"+ ),+ f"WebCryptoAPI/example.window.html?{WINDOW_JS_WINDOW_QUERY}#frag",+ )+ def test_any_js_metadata_matches_wpt_header_rules(self) -> None:+ meta = parse_any_js_meta(+ """//META: variant=?first+// META: script=helper.js++// META: variant=?ignored+"""+ )++ self.assertEqual(meta.variants, ["?first"])+ self.assertEqual(meta.scripts, ["helper.js"])diff --git a/moli-benchmark/tests/test_wpt_cross_case_lists.py b/moli-benchmark/tests/test_wpt_cross_case_lists.pynew file mode 100644index 000000000..5543f95d4--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_case_lists.py@@ -0,0 +1,463 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossCaseListsTests(WptCrossTestCase):+ def test_repo_case_lists_are_overwritten_for_primary_engine(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ out_dir = Path(temp_dir)+ matrix = [+ {+ "case_path": "c/timeout.html",+ "results": {+ "chrome": {"status": "pass"},+ "moli": {"status": "timeout"},+ },+ },+ {+ "case_path": "b/fail.html",+ "results": {+ "chrome": {"status": "pass"},+ "moli": {"status": "fail"},+ },+ },+ {+ "case_path": "a/pass.html",+ "results": {+ "chrome": {"status": "fail"},+ "moli": {"status": "pass"},+ },+ },+ {+ "case_path": "d/crash.html",+ "results": {+ "chrome": {"status": "pass"},+ "moli": {"status": "crash"},+ },+ },+ {+ "case_path": "f/error.html",+ "results": {+ "chrome": {"status": "pass"},+ "moli": {"status": "error"},+ },+ },+ {+ "case_path": "e/stalled.html",+ "results": {+ "chrome": {"status": "pass"},+ "moli": {"status": "harness-stalled"},+ },+ },+ {+ "case_path": "g/missing.html",+ "results": {+ "chrome": {"status": "pass"},+ },+ },+ {+ "case_path": "h/unknown.html",+ "results": {+ "chrome": {"status": "pass"},+ "moli": {"status": "unknown"},+ },+ },+ ]++ _write_repo_case_lists(+ matrix,+ ["chrome", "moli"],+ case_list_dir=out_dir,+ )++ expected = {+ "passed-cases.txt": "a/pass.html\n",+ "failed-cases.txt": "b/fail.html\n",+ "timeout-cases.txt": "c/timeout.html\n",+ "crash-cases.txt": "d/crash.html\n",+ "harness-stalled-cases.txt": "e/stalled.html\n",+ "error-cases.txt": "f/error.html\n",+ "missing-cases.txt": "g/missing.html\n",+ "other-cases.txt": "h/unknown.html\n",+ }+ for file_name, content in expected.items():+ self.assertEqual((out_dir / file_name).read_text(encoding="utf-8"), content)++ _write_repo_case_lists(+ [+ {+ "case_path": "only/pass.html",+ "results": {"moli": {"status": "pass"}},+ }+ ],+ ["moli"],+ case_list_dir=out_dir,+ )++ self.assertEqual(+ (out_dir / "passed-cases.txt").read_text(encoding="utf-8"),+ "only/pass.html\n",+ )+ for file_name in CASE_LIST_FILES.values():+ expected_content = "only/pass.html\n" if file_name == "passed-cases.txt" else ""+ self.assertEqual(+ (out_dir / file_name).read_text(encoding="utf-8"),+ expected_content,+ )+ def test_repo_case_lists_only_refresh_for_full_runs(self) -> None:+ parser = _build_parser()+ full = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ ]+ )+ explicit_case = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--case",+ "custom-elements/Document-createElement.html",+ ]+ )+ dir_prefix = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--dir-prefix",+ "shadow-dom",+ ]+ )+ limited = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--limit",+ "1",+ ]+ )+ tentative = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--include-tentative",+ ]+ )+ any_js = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--any-js-global",+ "window",+ ]+ )+ layout = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--profile",+ "layout",+ ]+ )+ all_profiles = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--profile",+ "all",+ ]+ )++ self.assertTrue(_is_full_case_list_run(full))+ self.assertTrue(_is_full_case_list_run(all_profiles))+ self.assertFalse(_is_full_case_list_run(layout))+ self.assertFalse(_is_full_case_list_run(explicit_case))+ self.assertFalse(_is_full_case_list_run(dir_prefix))+ self.assertFalse(_is_full_case_list_run(limited))+ self.assertFalse(_is_full_case_list_run(tentative))+ self.assertFalse(_is_full_case_list_run(any_js))+ def test_main_does_not_refresh_repo_case_lists_for_non_full_runs(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ manifest = root / "known.json"+ output_dir = root / "out"+ repo_case_lists = root / "repo-case-lists"+ repo_case_lists.mkdir()+ (repo_case_lists / "passed-cases.txt").write_text(+ "existing/full-case.html\n",+ encoding="utf-8",+ )+ manifest.write_text(+ json.dumps({"engine": "moli", "rules": []}),+ encoding="utf-8",+ )++ code = self._run_wpt_cross_with_fake_moli(+ output_dir=output_dir,+ known_failures=manifest,+ case_status="pass",+ extra_args=["--limit", "1"],+ )++ self.assertEqual(code, 0)+ self.assertEqual(+ (repo_case_lists / "passed-cases.txt").read_text(encoding="utf-8"),+ "existing/full-case.html\n",+ )+ self.assertFalse((repo_case_lists / "failed-cases.txt").exists())+ def test_case_origin_classification_preserves_secure_context_signal(self) -> None:+ worker_case = any_js_case_path_for_global(+ "WebCryptoAPI/idlharness.https.any.js",+ ANY_JS_DEDICATED_WORKER_GLOBAL,+ )+ secure_window_case = any_js_case_path_for_global(+ "WebCryptoAPI/digest/digest.https.any.js",+ ANY_JS_WINDOW_GLOBAL,+ )+ nonsecure_window_case = any_js_case_path_for_global(+ "WebCryptoAPI/historical.any.js",+ ANY_JS_WINDOW_GLOBAL,+ )++ self.assertTrue(_case_requires_trustworthy_origin(worker_case))+ self.assertTrue(_case_requires_trustworthy_origin(secure_window_case))+ self.assertFalse(_case_requires_trustworthy_origin(nonsecure_window_case))+ def test_url_for_case_origin_uses_non_loopback_only_for_secure_context_negative_cases(+ self,+ ) -> None:+ class FakeServer:+ external_base_url = "http://[2001:db8::1]:9000"++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ base = self.external_base_url if external else "http://127.0.0.1:8000"+ return f"{base}/{case_path}"++ secure_case = any_js_case_path_for_global(+ "WebCryptoAPI/digest/digest.https.any.js",+ ANY_JS_WINDOW_GLOBAL,+ )+ nonsecure_case = any_js_case_path_for_global(+ "WebCryptoAPI/historical.any.js",+ ANY_JS_WINDOW_GLOBAL,+ )+ secure_context_negative_case = "secure-contexts/basic-shared-worker.html"+ audio_output_negative_case = "audio-output/secure-context.html"+ credential_management_negative_case = (+ "credential-management/require_securecontext.html"+ )+ explicit_http_case = "digital-credentials/non-secure-contexts.http.html"+ underscore_insecure_context_case = "web-nfc/nfc_insecure_context.html"+ pointer_event_negative_case = "pointerevents/pointerevent_constructor.html"+ host_sensitive_case = "webmessaging/with-ports/020.html"++ self.assertTrue(_case_requires_trustworthy_origin(secure_case))+ self.assertFalse(_case_requires_trustworthy_origin(nonsecure_case))+ self.assertTrue(+ _case_requires_non_trustworthy_origin(secure_context_negative_case)+ )+ self.assertTrue(_case_requires_non_trustworthy_origin(audio_output_negative_case))+ self.assertTrue(+ _case_requires_non_trustworthy_origin(credential_management_negative_case)+ )+ self.assertTrue(_case_requires_non_trustworthy_origin(explicit_http_case))+ self.assertTrue(+ _case_requires_non_trustworthy_origin(underscore_insecure_context_case)+ )+ self.assertTrue(_case_requires_non_trustworthy_origin(pointer_event_negative_case))+ self.assertFalse(_case_requires_non_trustworthy_origin(host_sensitive_case))+ self.assertEqual(+ _url_for_case_origin(FakeServer(), secure_case, external=False),+ f"http://127.0.0.1:8000/{secure_case}",+ )+ self.assertEqual(+ _url_for_case_origin(FakeServer(), nonsecure_case, external=False),+ f"http://127.0.0.1:8000/{nonsecure_case}",+ )+ self.assertEqual(+ _url_for_case_origin(+ FakeServer(), secure_context_negative_case, external=False+ ),+ f"http://[2001:db8::1]:9000/{secure_context_negative_case}",+ )+ self.assertEqual(+ _url_for_case_origin(FakeServer(), audio_output_negative_case, external=False),+ f"http://[2001:db8::1]:9000/{audio_output_negative_case}",+ )+ self.assertEqual(+ _url_for_case_origin(+ FakeServer(), credential_management_negative_case, external=False+ ),+ f"http://[2001:db8::1]:9000/{credential_management_negative_case}",+ )+ self.assertEqual(+ _url_for_case_origin(FakeServer(), explicit_http_case, external=False),+ f"http://[2001:db8::1]:9000/{explicit_http_case}",+ )+ self.assertEqual(+ _url_for_case_origin(+ FakeServer(), underscore_insecure_context_case, external=False+ ),+ f"http://[2001:db8::1]:9000/{underscore_insecure_context_case}",+ )+ self.assertEqual(+ _url_for_case_origin(+ FakeServer(), pointer_event_negative_case, external=False+ ),+ f"http://[2001:db8::1]:9000/{pointer_event_negative_case}",+ )+ self.assertEqual(+ _url_for_case_origin(FakeServer(), host_sensitive_case, external=False),+ f"http://127.0.0.1:8000/{host_sensitive_case}",+ )+ self.assertEqual(+ _url_for_case_origin(FakeServer(), secure_case, external=True),+ f"http://[2001:db8::1]:9000/{secure_case}",+ )+ def test_cli_mode_preserves_case_origin_classification(self) -> None:+ calls = []++ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.base_url = "http://127.0.0.1:8000"+ self.alternate_base_url = "http://127.0.0.1:8001"+ self.external_base_url = "http://[2001:db8::1]:9000"+ self.external_alternate_base_url = "http://[2001:db8::1]:9001"+ self.external_remote_base_url = "http://[2001:db8::1]:9002"+ self.external_host = "2001:db8::1"++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ return None++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ base = self.external_base_url if external else self.base_url+ return f"{base}/{case_path}"++ result_dict = {+ "engine": "moli",+ "binary": "/tmp/moli",+ "binary_sha256": "sha",+ "binary_version": "0.1.0",+ "endpoint": "cli:/tmp/moli",+ "ready_ms": None,+ "setup_error": None,+ "cases": [],+ }++ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir) / "out"++ def fake_run_engine_on_cases_cli(**kwargs: object) -> SimpleNamespace:+ calls.append(kwargs["cases"])+ return SimpleNamespace(setup_error=None)++ with (+ patch(+ "moli_benchmark.wpt_cross.__main__.enumerate_cases",+ return_value=[+ WptCase("secure-contexts/basic-shared-worker.html"),+ WptCase("digital-credentials/non-secure-contexts.http.html"),+ WptCase("WebCryptoAPI/digest/digest.https.html"),+ ],+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.build_driver",+ return_value=SimpleNamespace(cli_fetch_command=["moli"]),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",+ side_effect=fake_run_engine_on_cases_cli,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",+ return_value=result_dict,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",+ Path(temp_dir) / "repo-case-lists",+ ),+ patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),+ redirect_stdout(StringIO()),+ redirect_stderr(StringIO()),+ ):+ code = main(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ str(output_dir),+ ]+ )++ self.assertEqual(code, 0)+ self.assertEqual(+ calls,+ [+ [+ (+ "secure-contexts/basic-shared-worker.html",+ "http://[2001:db8::1]:9000/secure-contexts/basic-shared-worker.html",+ WPT_CROSS_CASE_TIMEOUT_SECONDS,+ 1.0,+ ),+ (+ "digital-credentials/non-secure-contexts.http.html",+ "http://[2001:db8::1]:9000/digital-credentials/non-secure-contexts.http.html",+ WPT_CROSS_CASE_TIMEOUT_SECONDS,+ 1.0,+ ),+ (+ "WebCryptoAPI/digest/digest.https.html",+ "http://127.0.0.1:8000/WebCryptoAPI/digest/digest.https.html",+ WPT_CROSS_CASE_TIMEOUT_SECONDS,+ 1.0,+ )+ ]+ ],+ )diff --git a/moli-benchmark/tests/test_wpt_cross_cli_runner.py b/moli-benchmark/tests/test_wpt_cross_cli_runner.pynew file mode 100644index 000000000..a0edb8a42--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_cli_runner.py@@ -0,0 +1,301 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossCliRunnerTests(WptCrossTestCase):+ def test_cli_runner_distinguishes_app_errors_from_process_crashes(self) -> None:+ self.assertEqual(_nonzero_exit_status(1, "Error: unsupported navigation"), "error")+ self.assertEqual(_nonzero_exit_status(-11, ""), "crash")+ self.assertEqual(_nonzero_exit_status(101, "thread 'main' panicked at x"), "crash")+ def test_cli_runner_stderr_tail_is_bounded(self) -> None:+ self.assertEqual(_stderr_tail("short"), "short")+ self.assertEqual(len(_stderr_tail("x" * 3000)), 2000)+ def test_cli_runner_uses_longer_payload_grace_only_after_successful_process(self) -> None:+ self.assertEqual(+ _payload_grace_for_process_result(+ proc_error=None,+ proc_returncode=0,+ payload_grace_seconds=2.0,+ successful_process_payload_grace_seconds=8.0,+ ),+ 8.0,+ )+ self.assertEqual(+ _payload_grace_for_process_result(+ proc_error="engine subprocess wall timeout",+ proc_returncode=None,+ payload_grace_seconds=2.0,+ successful_process_payload_grace_seconds=5.0,+ ),+ 2.0,+ )+ self.assertEqual(+ _payload_grace_for_process_result(+ proc_error=None,+ proc_returncode=1,+ payload_grace_seconds=2.0,+ successful_process_payload_grace_seconds=5.0,+ ),+ 2.0,+ )+ def test_cli_runner_keeps_final_payload_status_after_process_timeout(self) -> None:+ class FakeResults:+ def wait_for_final(self, key: str, timeout: float) -> dict:+ self.wait_key = key+ return {+ "source": "completion-callback",+ "harness": {"status": 0, "message": None},+ "tests": [{"name": "done", "status": 0, "message": None}],+ }++ def get(self, key: str) -> None:+ return None++ fixture_server = SimpleNamespace(+ results=FakeResults(),+ )++ result = _classify_cli_case_result(+ case_path="case.html",+ url="http://example.test/case.html",+ bridge_key="/case.html",+ fixture_server=fixture_server,+ subprocess_result=_CliSubprocessResult(+ duration_ms=10.0,+ proc_error="engine subprocess wall timeout after 0.0s",+ proc_returncode=None,+ proc_stderr="",+ proc_stdout=b"",+ wait_script_timeout=False,+ ),+ payload_grace_seconds=0,+ successful_process_payload_grace_seconds=8.0,+ )++ self.assertEqual(result.status, "pass")+ self.assertIsNone(result.error)+ self.assertEqual(result.payload_source, "completion-callback")+ def test_cli_runner_defaults_harness_timeout_independent_of_process_deadline(self) -> None:+ class FakeFuture:+ def __init__(self, result: CaseResult) -> None:+ self._result = result++ def result(self) -> CaseResult:+ return self._result++ class FakeProcessPoolExecutor:+ instances: list["FakeProcessPoolExecutor"] = []++ def __init__(self, *, max_workers: int, mp_context: object) -> None:+ self.max_workers = max_workers+ self.mp_context = mp_context+ self.submitted = []+ self.instances.append(self)++ def __enter__(self) -> "FakeProcessPoolExecutor":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def submit(self, fn, job):+ self.submitted.append((fn, job))+ return FakeFuture(fn(job))++ def fake_as_completed(futures):+ return list(futures)++ def fake_worker(job):+ return CaseResult(+ case_path=job.case_path,+ url=f"http://worker.test/{job.case_path}",+ status="pass",+ duration_ms=1.0,+ )++ driver = SimpleNamespace(+ name="moli",+ version_args=["--version"],+ extra_env={},+ cli_fetch_command=lambda binary, url, timeout: [str(binary), "fetch", url],+ resolve_binary=lambda override: Path("/tmp/moli"),+ )+ fixture_server = SimpleNamespace(+ external_host="2001:db8::1",+ external_base_url="http://[2001:db8::1]:9000",+ wpt_root=Path("/tmp/wpt"),+ )++ def fake_subprocess_run(argv, **kwargs):+ self.assertEqual(argv, ["/tmp/moli", "--version"])+ return SimpleNamespace(stdout="moli 0\n", stderr="", returncode=0)++ with (+ patch("moli_benchmark.wpt_cross.cli_runner.sha256_file", return_value="sha"),+ patch("moli_benchmark.wpt_cross.cli_runner.subprocess.run", fake_subprocess_run),+ patch(+ "moli_benchmark.wpt_cross.cli_runner.ProcessPoolExecutor",+ FakeProcessPoolExecutor,+ ),+ patch("moli_benchmark.wpt_cross.cli_runner.as_completed", fake_as_completed),+ patch("moli_benchmark.wpt_cross.cli_runner._run_cli_case_worker", fake_worker),+ ):+ result = run_engine_on_cases_cli(+ driver=driver,+ fixture_server=fixture_server,+ cases=[+ (+ "normal.html",+ "http://[2001:db8::1]:9000/normal.html",+ 120.0,+ ),+ (+ "long.html",+ "http://[2001:db8::1]:9000/long.html",+ 120.0,+ LONG_TIMEOUT_MULTIPLIER,+ ),+ ],+ parallelism=1,+ progress_every=0,+ )++ pool = FakeProcessPoolExecutor.instances[0]+ self.assertEqual(pool.max_workers, 1)+ self.assertEqual(pool.mp_context.get_start_method(), "spawn")+ self.assertEqual(pool.submitted[0][1].case_path, "normal.html")+ self.assertTrue(pool.submitted[0][1].external)+ self.assertEqual(pool.submitted[0][1].timeout_seconds, 120.0)+ self.assertEqual(pool.submitted[0][1].harness_timeout_multiplier, 1.0)+ self.assertEqual(+ pool.submitted[1][1].harness_timeout_multiplier,+ LONG_TIMEOUT_MULTIPLIER,+ )+ self.assertTrue(all(case.status == "pass" for case in result.cases))+ self.assertEqual(result.shutdown_info["scheduler"], "process-pool")+ def test_moli_cli_worker_appends_hardcoded_wpt_user_agent(self) -> None:+ captured_argv = []++ class FakeResults:+ def clear(self, key: str) -> None:+ return None++ def wait_for_final(self, key: str, timeout: float) -> dict:+ return {+ "source": "completion-callback",+ "harness": {"status": 0, "message": None},+ "tests": [{"name": "done", "status": 0, "message": None}],+ }++ def get(self, key: str) -> None:+ return None++ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.results = FakeResults()+ self.external_host = None++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ return None++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ return f"http://127.0.0.1:8000/{case_path}"++ driver = SimpleNamespace(+ name="moli",+ cli_fetch_command=lambda binary, url, timeout: [str(binary), "fetch", url],+ )++ def fake_run_cli_subprocess(argv, env, proc_timeout):+ captured_argv.append(argv)+ return _CliSubprocessResult(+ duration_ms=1.0,+ proc_error=None,+ proc_returncode=0,+ proc_stderr="",+ proc_stdout=b"",+ wait_script_timeout=False,+ )++ with (+ patch(+ "moli_benchmark.wpt_cross.cli_runner.build_driver",+ return_value=driver,+ ),+ patch(+ "moli_benchmark.wpt_cross.cli_runner.WptFixtureServer",+ FakeServer,+ ),+ patch(+ "moli_benchmark.wpt_cross.cli_runner._run_cli_subprocess",+ fake_run_cli_subprocess,+ ),+ ):+ result = _run_cli_case_worker(+ _CliCaseWorkerInput(+ engine="moli",+ binary="/tmp/moli",+ wpt_root="/tmp/wpt",+ case_path="case.html",+ external=False,+ timeout_seconds=8.0,+ harness_timeout_multiplier=1.0,+ env={},+ process_timeout_margin_seconds=4.0,+ payload_grace_seconds=0.0,+ successful_process_payload_grace_seconds=0.0,+ )+ )++ self.assertEqual(result.status, "pass")+ self.assertEqual(+ captured_argv,+ [+ [+ "/tmp/moli",+ "fetch",+ "http://127.0.0.1:8000/case.html",+ "--user-agent",+ MOLI_WPT_USER_AGENT,+ ]+ ],+ )+ def test_moli_cli_resolves_external_fixture_template_hosts(self) -> None:+ fixture_server = SimpleNamespace(+ external_host="2001:db8::42",+ external_port=12345,+ external_alternate_port=23456,+ external_remote_port=23456,+ )++ args = _moli_fixture_host_resolve_args(fixture_server)++ self.assertIn(+ "--http-host-resolve",+ args,+ )+ self.assertIn(+ "alt.localhost:12345:[2001:db8::42]",+ args,+ )+ self.assertIn(+ "www.localhost:23456:[2001:db8::42]",+ args,+ )+ self.assertEqual(+ args.count("localhost:23456:[2001:db8::42]"),+ 1,+ )diff --git a/moli-benchmark/tests/test_wpt_cross_fetch_fixtures.py b/moli-benchmark/tests/test_wpt_cross_fetch_fixtures.pynew file mode 100644index 000000000..0fdbf2a6b--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_fetch_fixtures.py@@ -0,0 +1,188 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossFetchFixturesTests(WptCrossTestCase):+ def test_fixture_server_parses_fetch_status_parameters_as_bytes(self) -> None:+ self.assertEqual(_fetch_status_response(""), (200, "OMG", "", b""))+ self.assertEqual(+ _fetch_status_response(+ "code=201&code=404&text=&text=ignored&type=text%2Fplain"+ "&content=%FF%FE%00%2B+end&content=ignored"+ ),+ (201, "", "text/plain", b"\xff\xfe\x00+ end"),+ )+ def test_fixture_server_models_fetch_status_for_supported_methods(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ for method in ["GET", "HEAD", "POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO"]:+ for status in [200, 204, 205, 304]:+ with self.subTest(method=method, status=status):+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.request(+ method,+ "/fetch/api/resources/status.py"+ f"?code={status}&text=Custom%20%FF"+ "&type=text%2Fplain%3Bcharset%3DUTF-16"+ "&content=%FF%FE%00%2B+end",+ body=b"ignored" if method not in {"GET", "HEAD"} else None,+ )+ response = connection.getresponse()+ self.assertEqual(response.status, status)+ self.assertEqual(response.reason, "Custom \xff")+ self.assertEqual(response.headers["X-Request-Method"], method)+ self.assertEqual(+ response.headers["Content-Type"],+ "text/plain;charset=UTF-16",+ )+ self.assertEqual(response.headers["Content-Length"], "8")+ self.assertEqual(+ response.read(),+ b"" if method == "HEAD" or status in {204, 304}+ else b"\xff\xfe\x00+ end",+ )+ finally:+ connection.close()+ def test_fixture_server_models_fetch_trickle_parameters_and_methods(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text("// testharness")+ with WptFixtureServer(root_path) as server, patch(+ "moli_benchmark.wpt_cross.server.time.sleep"+ ) as sleep:+ cases = [+ ("", 50, "text/plain", 0.5),+ ("count=2&count=9&ms=1.5&ms=900", 2, "text/plain", 0.0015),+ ("count=0&ms=0¬ype", 0, None, 0.0),+ ("count=-1&ms=0¬ype=false", 0, None, 0.0),+ ]+ for method in ["GET", "HEAD", "POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO"]:+ for query, count, mime, delay in cases:+ with self.subTest(method=method, query=query):+ sleep.reset_mock()+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.request(+ method, "/fetch/api/resources/trickle.py?" + query,+ body=b"upload" if method not in {"GET", "HEAD"} else None,+ )+ response = connection.getresponse()+ self.assertEqual(response.status, 200)+ self.assertEqual(response.headers.get("Content-Type"), mime)+ self.assertIsNone(response.headers.get("Content-Length"))+ self.assertIsNone(response.headers.get("Transfer-Encoding"))+ self.assertEqual(response.read(), b"" if method == "HEAD" else b"TEST_TRICKLE\n" * count)+ expected_sleeps = 1 if method == "HEAD" else count + 2+ self.assertEqual([args.args[0] for args in sleep.call_args_list], [delay] * expected_sleeps)+ finally:+ connection.close()+ for query in ["count=invalid", "count=", "ms=invalid", "ms=-1", "ms=nan", "ms=inf"]:+ with self.subTest(query=query):+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.request("GET", "/fetch/api/resources/trickle.py?" + query)+ response = connection.getresponse()+ self.assertEqual(response.status, 500)+ response.read()+ finally:+ connection.close()+ def test_fixture_server_delivers_fetch_trickle_headers_and_chunks_before_eof(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text("// testharness")+ headers_ready, chunk_ready = threading.Event(), threading.Event()+ release_first, release_second = threading.Event(), threading.Event()+ sleeps = []++ def pause(delay: float) -> None:+ sleeps.append(delay)+ if len(sleeps) == 2:+ headers_ready.set()+ self.assertTrue(release_first.wait(2))+ elif len(sleeps) == 3:+ chunk_ready.set()+ self.assertTrue(release_second.wait(2))++ with WptFixtureServer(root_path) as server, patch(+ "moli_benchmark.wpt_cross.server.time.sleep", side_effect=pause+ ):+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.request("GET", "/fetch/api/resources/trickle.py?count=2¬ype")+ response = connection.getresponse()+ self.assertTrue(headers_ready.wait(2))+ self.assertIsNone(response.headers.get("Content-Type"))+ release_first.set()+ self.assertEqual(response.read(13), b"TEST_TRICKLE\n")+ self.assertTrue(chunk_ready.wait(2))+ release_second.set()+ self.assertEqual(response.read(), b"TEST_TRICKLE\n")+ self.assertEqual(sleeps, [0.5] * 4)+ finally:+ release_first.set()+ release_second.set()+ connection.close()+ def test_fixture_server_reads_upload_before_starting_fetch_trickle(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text("// testharness")+ started = threading.Event()+ with WptFixtureServer(root_path) as server, patch(+ "moli_benchmark.wpt_cross.server.time.sleep",+ side_effect=lambda _: started.set(),+ ):+ for framing, first, last in [+ (b"Content-Length: 6", b"abc", b"def"),+ (b"Transfer-Encoding: chunked", b"3\r\nabc\r\n", b"3\r\ndef\r\n0\r\n\r\n"),+ ]:+ with self.subTest(framing=framing), socket.create_connection(("127.0.0.1", server.port), timeout=2) as connection:+ started.clear()+ connection.sendall(+ b"POST /fetch/api/resources/trickle.py?count=1&ms=0 HTTP/1.1\r\n"+ b"Host: localhost\r\n" + framing + b"\r\n\r\n" + first+ )+ self.assertFalse(started.wait(0.05), "response started before upload completed")+ connection.sendall(last)+ self.assertTrue(started.wait(2))+ response = connection.makefile("rb").read()+ self.assertEqual(response.split(b"\r\n\r\n", 1)[1], b"TEST_TRICKLE\n")+ def test_fixture_server_counts_link_stylesheet_requests_and_clears_on_read(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text("// testharness", encoding="utf-8")+ resource = "/html/semantics/document-metadata/the-link-element/stylesheet.py"+ fixture = root_path / resource.lstrip("/")+ fixture.parent.mkdir(parents=True)+ fixture.write_text("# dynamic stylesheet resource", encoding="utf-8")+ first = "11111111-1111-4111-8111-111111111111"+ second = "22222222-2222-4222-8222-222222222222"+ with WptFixtureServer(root_path) as server:+ def request(query: str, *, method: str = "GET", alternate: bool = False) -> tuple[str, bytes]:+ port = server.alternate_port if alternate else server.port+ url = f"http://127.0.0.1:{port}{resource}?{query}"+ with urlopen(Request(url, method=method), timeout=2) as response:+ self.assertEqual(response.status, 200)+ self.assertIsNone(response.headers.get("Cache-Control"))+ return response.headers["Content-Type"], response.read()++ self.assertEqual(request(f"id={first}&count="), ("text/html", b"0"))+ self.assertEqual(request(f"id={first}"), ("text/css", b"body {color: red;}"))+ self.assertEqual(request(f"id={first}", alternate=True), ("text/css", b"body {color: red;}"))+ self.assertEqual(request(f"id={first}", method="HEAD"), ("text/css", b""))+ self.assertEqual(request(f"id={second}"), ("text/css", b"body {color: red;}"))+ self.assertEqual(request(f"id={first}&count=foo"), ("text/html", b"3"))+ self.assertEqual(request(f"id={second}&count=foo"), ("text/html", b"1"))+ self.assertEqual(request(f"id={first}&count=foo"), ("text/html", b"0"))+ self.assertEqual(request(f"id={first}"), ("text/css", b"body {color: red;}"))+ self.assertEqual(request(f"id={first}&count=foo"), ("text/html", b"1"))diff --git a/moli-benchmark/tests/test_wpt_cross_fixture_resources.py b/moli-benchmark/tests/test_wpt_cross_fixture_resources.pynew file mode 100644index 000000000..034612688--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_fixture_resources.py@@ -0,0 +1,594 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossFixtureResourcesTests(WptCrossTestCase):+ def test_fixture_server_models_reporting_resource_stash(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ report_url = f"{server.base_url}/reporting/resources/report.py?op=put&reportID=abc"+ payload = json.dumps(+ {"csp-report": {"violated-directive": "frame-src 'none'"}}+ ).encode("utf-8")+ request = Request(+ report_url,+ data=payload,+ headers={"Content-Type": "application/csp-report"},+ method="POST",+ )+ with urlopen(request, timeout=2) as response:+ self.assertEqual(response.status, 200)++ retrieve_url = (+ f"{server.base_url}/reporting/resources/report.py"+ "?op=retrieve_report&timeout=0&reportID=abc"+ )+ with urlopen(retrieve_url, timeout=2) as response:+ reports = json.loads(response.read().decode("utf-8"))++ self.assertEqual(+ reports[0]["csp-report"]["violated-directive"],+ "frame-src 'none'",+ )+ self.assertEqual(+ reports[0]["metadata"]["content_type"],+ "application/csp-report",+ )+ def test_fixture_server_accepts_standard_websocket_echo_endpoint(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ with socket.create_connection(+ ("127.0.0.1", server.port), timeout=2+ ) as connection:+ connection.sendall(+ b"GET /echo HTTP/1.1\r\n"+ b"Host: localhost\r\n"+ b"Connection: Upgrade\r\n"+ b"Upgrade: websocket\r\n"+ b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"+ b"Sec-WebSocket-Version: 13\r\n\r\n"+ )+ response = b""+ while b"\r\n\r\n" not in response:+ response += connection.recv(4096)+ self.assertTrue(response.startswith(b"HTTP/1.0 101"))+ self.assertIn(b"Upgrade: websocket\r\n", response)+ self.assertIn(+ b"Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n",+ response,+ )++ mask = b"\x01\x02\x03\x04"+ payload = b"ping"+ masked = bytes(+ value ^ mask[index % len(mask)]+ for index, value in enumerate(payload)+ )+ connection.sendall(b"\x81\x84" + mask + masked)+ echoed = b""+ while len(echoed) < 6:+ echoed += connection.recv(6 - len(echoed))+ self.assertEqual(echoed, b"\x81\x04ping")+ def test_fixture_server_serves_raw_asis_header_bytes(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ (root_path / "raw.asis").write_bytes(+ b"HTTP/1.1 200 OK\r\n"+ b"Content-Type: text/plain\r\n"+ b"X-Custom-Header-Bytes: \xe2\x80\xa6\r\n"+ b"\r\n"+ b"OK"+ )++ with WptFixtureServer(root_path) as server:+ with urlopen(f"{server.base_url}/raw.asis", timeout=2) as response:+ self.assertEqual(response.read(), b"OK")+ header = response.headers["X-Custom-Header-Bytes"]++ self.assertEqual(header.encode("latin-1"), b"\xe2\x80\xa6")+ def test_fixture_server_serves_raw_sidecar_header_bytes(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ (root_path / "manifest.webmanifest").write_bytes(b"{}")+ (root_path / "manifest.webmanifest.headers").write_bytes(+ b"Content-Type: \xc3\x97\xc2\xba invalid\r\n"+ )++ with WptFixtureServer(root_path) as server:+ with urlopen(+ f"{server.base_url}/manifest.webmanifest",+ timeout=2,+ ) as response:+ self.assertEqual(response.read(), b"{}")+ header = response.headers["Content-Type"]++ self.assertEqual(header.encode("latin-1"), b"\xc3\x97\xc2\xba invalid")+ def test_fixture_server_uses_explicit_content_type_header_as_override(self) -> None:+ self.assertEqual(+ _static_response_header_block(+ "application/octet-stream",+ [("Content-Type", "application/wasm"), ("X-Test", "ok")],+ ),+ [("Content-Type", "application/wasm"), ("X-Test", "ok")],+ )+ self.assertEqual(+ _static_response_header_block("application/javascript", [("X-Test", "ok")]),+ [("Content-Type", "application/javascript"), ("X-Test", "ok")],+ )+ def test_fixture_server_detects_explicit_content_length_header(self) -> None:+ header_block = _static_response_header_block(+ "text/html",+ [("Content-Length", "403"), ("X-Test", "ok")],+ )++ self.assertTrue(_headers_include(header_block, "content-length"))+ def test_fixture_server_preserves_explicit_content_length_body_boundary(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness",+ encoding="utf-8",+ )+ body = (+ b'<!doctype html><script src="/resources/testharnessreport.js"></script>'+ b"<body>PASS"+ )+ fixture = root_path / "content-length.html"+ fixture.write_bytes(body + b"FAIL")+ fixture.with_name("content-length.html.headers").write_text(+ f"Content-Length: {len(body)}\n",+ encoding="utf-8",+ )++ with WptFixtureServer(root_path) as server:+ server.set_harness_timeout_multipliers(+ {"content-length.html": 12.0}+ )+ with urlopen(+ f"{server.base_url}/content-length.html",+ timeout=2,+ ) as response:+ served = response.read()++ self.assertEqual(served, body)+ self.assertNotIn(BENCH_TIMEOUT_MULTIPLIER_QUERY.encode("ascii"), served)+ def test_fixture_server_maps_legacy_webidl_parser_resource(self) -> None:+ self.assertEqual(+ _legacy_wpt_resource_alias("/resources/WebIDLParser.js"),+ "resources/webidl2/lib/webidl2.js",+ )+ self.assertIsNone(_legacy_wpt_resource_alias("/resources/testharness.js"))+ def test_fixture_server_builds_any_js_window_wrapper(self) -> None:+ body = _any_js_window_wrapper(+ "/WebCryptoAPI/sign_verify/hmac.https.any.html",+ b"// META: title=WebCryptoAPI hmac\n"+ b"// META: script=../util/helpers.js\n"+ b"// META: script=hmac_vectors.js\n"+ b"// META: timeout=long\n"+ b"run_test();\n",+ )++ self.assertIn(b'<meta name="timeout" content="long">', body)+ self.assertIn(b"<title>WebCryptoAPI hmac</title>", body)+ self.assertIn(b"self.GLOBAL = {", body)+ self.assertIn(b'<script src="/resources/testharness.js"></script>', body)+ self.assertIn(b'<script src="/resources/testharnessreport.js"></script>', body)+ self.assertIn(b'<script src="../util/helpers.js"></script>', body)+ self.assertIn(b'<script src="hmac_vectors.js"></script>', body)+ self.assertIn(b'<div id="log"></div>', body)+ self.assertIn(b'<script src="hmac.https.any.js"></script>', body)+ def test_fixture_server_builds_window_js_wrapper_without_any_global(self) -> None:+ body = _window_js_window_wrapper(+ "/WebCryptoAPI/algorithm-discards-context.https.window.html",+ b"// META: title=Window case\n"+ b"// META: script=helper.js\n"+ b"test(() => {}, 'ok');\n",+ )++ self.assertIn(b"<title>Window case</title>", body)+ self.assertIn(b'<script src="/resources/testharness.js"></script>', body)+ self.assertIn(b'<script src="/resources/testharnessreport.js"></script>', body)+ self.assertIn(b'<script src="helper.js"></script>', body)+ self.assertIn(b'<script src="algorithm-discards-context.https.window.js"></script>', body)+ self.assertNotIn(b"self.GLOBAL", body)+ def test_fixture_server_builds_wpt_any_js_query_wrapper(self) -> None:+ html = _wpt_any_window_wrapper_html(+ "wasm/jsapi/feature.any.js",+ "// META: script=../support/helper.js\n// META: script=/common/gc.js?run=1\n",+ query="variant=1&moli-wpt-any=window",+ )++ self.assertIn('<script src="/resources/testharness.js"></script>', html)+ self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)+ self.assertIn('<script src="/wasm/support/helper.js"></script>', html)+ self.assertIn('<script src="/common/gc.js?run=1"></script>', html)+ self.assertIn(+ '<script src="/wasm/jsapi/feature.any.js?variant=1"></script>',+ html,+ )+ self.assertNotIn("moli-wpt-any", html)+ def test_fixture_server_builds_any_js_dedicated_worker_wrapper(self) -> None:+ html = _wpt_any_dedicated_worker_wrapper_html(+ "wasm/jsapi/feature.any.js",+ query="variant=1&moli-wpt-any=dedicatedworker",+ )++ self.assertIn('<script src="/resources/testharness.js"></script>', html)+ self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)+ self.assertIn(+ 'fetch_tests_from_worker(new Worker("/wasm/jsapi/feature.any.worker.js?variant=1"))',+ html,+ )+ self.assertNotIn("moli-wpt-any", html)+ def test_fixture_server_builds_any_js_dedicated_worker_script(self) -> None:+ js = _wpt_any_dedicated_worker_wrapper_js(+ "wasm/jsapi/feature.any.js",+ "// META: script=../support/helper.js\n// META: script=/common/gc.js?run=1\n",+ query="variant=1&moli-wpt-any=dedicatedworker",+ )++ self.assertIn("isWindow:function(){return false;}", js)+ self.assertIn("isWorker:function(){return true;}", js)+ self.assertIn('importScripts("/resources/testharness.js");', js)+ self.assertIn('importScripts("/wasm/support/helper.js");', js)+ self.assertIn('importScripts("/common/gc.js?run=1");', js)+ self.assertIn('importScripts("/wasm/jsapi/feature.any.js?variant=1");', js)+ self.assertTrue(js.rstrip().endswith("done();"))+ self.assertNotIn("moli-wpt-any", js)+ def test_fixture_server_builds_window_js_wrapper(self) -> None:+ html = _wpt_window_js_wrapper_html(+ "wasm/serialization/transfer.window.js",+ "// META: script=../support/helper.js\n// META: script=/common/gc.js?run=1\n",+ query="variant=1&moli-wpt-script=window",+ )++ self.assertIn('<script src="/resources/testharness.js"></script>', html)+ self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)+ self.assertIn('<script src="/wasm/support/helper.js"></script>', html)+ self.assertIn('<script src="/common/gc.js?run=1"></script>', html)+ self.assertIn(+ '<script src="/wasm/serialization/transfer.window.js?variant=1"></script>',+ html,+ )+ self.assertNotIn("moli-wpt-script", html)+ def test_fixture_server_builds_worker_js_wrapper(self) -> None:+ html = _wpt_dedicated_worker_js_wrapper_html(+ "wasm/create_multiple_memory.worker.js",+ query="variant=1&moli-wpt-script=dedicatedworker",+ )++ self.assertIn('<script src="/resources/testharness.js"></script>', html)+ self.assertIn('<script src="/resources/testharnessreport.js"></script>', html)+ self.assertIn(+ 'fetch_tests_from_worker(new Worker("/wasm/create_multiple_memory.worker.js?variant=1"))',+ html,+ )+ self.assertNotIn("moli-wpt-script", html)+ def test_any_js_worker_script_path_round_trips_to_source_path(self) -> None:+ worker_path = any_js_worker_script_path(+ "wasm/jsapi/feature.any.js?variant=1"+ )++ self.assertEqual(+ worker_path,+ "wasm/jsapi/feature.any.worker.js?variant=1",+ )+ self.assertEqual(+ any_js_source_script_path(worker_path),+ "wasm/jsapi/feature.any.js?variant=1",+ )+ def test_any_js_case_path_for_global_replaces_existing_wrapper_query(self) -> None:+ self.assertEqual(+ any_js_case_path_for_global(+ "wasm/jsapi/feature.any.js?moli-wpt-any=window&variant=1",+ ANY_JS_DEDICATED_WORKER_GLOBAL,+ ),+ "wasm/jsapi/feature.any.js?variant=1&moli-wpt-any=dedicatedworker",+ )+ def test_fixture_server_resolves_any_js_meta_scripts_within_wpt_root(self) -> None:+ self.assertEqual(+ _resolve_wpt_static_script_url(+ "wasm/jsapi/feature.any.js",+ "../support/helper.js?mode=1",+ ),+ "/wasm/support/helper.js?mode=1",+ )+ self.assertEqual(+ _resolve_wpt_static_script_url(+ "wasm/jsapi/feature.any.js",+ "/resources/WebIDLParser.js",+ ),+ "/resources/WebIDLParser.js",+ )+ self.assertIsNone(+ _resolve_wpt_static_script_url(+ "wasm/jsapi/feature.any.js",+ "../../../escape.js",+ )+ )+ def test_fixture_server_substitutes_core_sub_template_variables(self) -> None:+ body = (+ b"http://{{domains[www2]}}:{{ports[http][0]}}/"+ b" location={{location[port]}}"+ b" scheme={{location[scheme]}}"+ b" hostname={{location[hostname]}}"+ b" server={{location[server]}}"+ b" path={{location[path]}}"+ b" alt={{ports[http][1]}}"+ b" same={{domains[www]}} host={{host}}"+ b" hosts={{hosts[][]}}/{{hosts[][www]}}/{{hosts[alt][]}}/{{hosts[alt][www]}}"+ b" https={{ports[https][0]}}/{{ports[https][1]}}"+ b" https-url=https://{{domains[www]}}:{{ports[https][0]}}/secure/"+ b" https-remote=https://{{domains[www1]}}:{{ports[https][0]}}/remote/"+ b" https-hosts-remote=https://{{hosts[][www]}}:{{ports[https][1]}}/cross/"+ b" https-hosts-bare=https://{{hosts[][]}}:{{ports[https][0]}}/bare/"+ b" https-hosts-alt=https://{{hosts[alt][]}}:{{ports[https][0]}}/alt/"+ b" https-hosts-alt-www=https://{{hosts[alt][www]}}:{{ports[https][1]}}/alt-www/"+ b" ws={{ports[ws][0]}}/{{ports[ws][1]}}"+ b" wss={{ports[wss][0]}}/{{ports[wss][1]}}"+ b" ws-url=ws://{{host}}:{{ports[ws][0]}}/socket"+ b" wss-url=wss://{{host}}:{{ports[wss][0]}}/socket"+ )++ self.assertEqual(+ _substitute_wpt_template_variables(+ body,+ port=12345,+ alternate_port=23456,+ request_path="/secure-contexts/server-locations.sub.js",+ request_hostname="example.test",+ ),+ b"http://www2.localhost:12345/ location=12345 scheme=http hostname=example.test"+ b" server=http://example.test:12345 path=/secure-contexts/server-locations.sub.js"+ b" alt=23456 same=www.localhost host=example.test"+ b" hosts=localhost/www.localhost/alt.localhost/www.alt.localhost https=12345/23456"+ b" https-url=http://example.test:12345/secure/"+ b" https-remote=http://example.test:23456/remote/"+ b" https-hosts-remote=http://www.localhost:23456/cross/"+ b" https-hosts-bare=http://localhost:12345/bare/"+ b" https-hosts-alt=http://alt.localhost:12345/alt/"+ b" https-hosts-alt-www=http://www.alt.localhost:23456/alt-www/"+ b" ws=12345/23456 wss=12345/23456"+ b" ws-url=ws://example.test:12345/socket"+ b" wss-url=ws://example.test:12345/socket",+ )+ def test_fixture_server_pipe_sub_requests_template_substitution(self) -> None:+ self.assertTrue(+ _needs_wpt_template_substitution(+ "sandboxed-tests.html",+ b'importScripts("http://{{host}}:{{ports[http][0]}}/resources/testharness.js");',+ "pipe=sub",+ )+ )+ self.assertTrue(+ _needs_wpt_template_substitution(+ "resource.html",+ b"{{host}}",+ "pipe=header(X-Test,yes)|sub",+ )+ )+ self.assertFalse(+ _needs_wpt_template_substitution(+ "resource.html",+ b"{{host}}",+ "",+ )+ )+ def test_fixture_server_maps_external_ipv6_domain_location_port_to_remote_port(+ self,+ ) -> None:+ body = (+ b"http://{{domains[www]}}:{{location[port]}}/a"+ b" http://{{domains[www2]}}:{{location[port]}}/b"+ b" http://{{domains[www]}}:{{ports[http][0]}}/c"+ b" http://{{domains[www1]}}:{{ports[http][0]}}/d"+ b" http://{{domains[www1]}}:{{ports[http][1]}}/e"+ )++ self.assertEqual(+ _substitute_wpt_template_variables(+ body,+ port=12345,+ alternate_port=23456,+ remote_port=34567,+ request_hostname="[2001:db8::1]",+ ),+ b"http://[2001:db8::1]:34567/a"+ b" http://[2001:db8::1]:34567/b"+ b" http://[2001:db8::1]:34567/c"+ b" http://[2001:db8::1]:34567/d"+ b" http://[2001:db8::1]:23456/e",+ )+ def test_fixture_server_substitutes_get_query_template_variables(self) -> None:+ body = (+ b"var expected_logs = {{GET[logs]}};"+ b" var timeout = \"{{GET[timeout]}}\";"+ b" var missing = \"{{GET[missing]}}\";"+ )++ self.assertEqual(+ _substitute_wpt_template_variables(+ body,+ port=12345,+ query='logs=["xhr allowed","TEST COMPLETE"]&timeout=2',+ ),+ b'var expected_logs = ["xhr allowed","TEST COMPLETE"];'+ b' var timeout = "2";'+ b' var missing = "";',+ )+ def test_fixture_server_substitutes_template_variables_in_sidecar_headers(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ fixture = Path(root) / "frame-ancestors.sub.html"+ fixture.write_text("<!doctype html>", encoding="utf-8")+ fixture.with_name("frame-ancestors.sub.html.sub.headers").write_text(+ "Content-Security-Policy: frame-ancestors {{GET[policy]}} {{location[scheme]}}://{{location[host]}}\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _static_response_headers(+ fixture,+ "policy=%27none%27",+ port=12345,+ alternate_port=23456,+ request_hostname="example.test",+ ),+ [+ (+ "Content-Security-Policy",+ "frame-ancestors 'none' http://example.test:12345",+ )+ ],+ )+ def test_fixture_server_substitution_preserves_non_utf8_bytes(self) -> None:+ body = b"\xff{{host}}\xfe{{ports[http][0]}}"++ self.assertEqual(+ _substitute_wpt_template_variables(body, port=12345),+ b"\xfflocalhost\xfe12345",+ )+ def test_fixture_server_distinguishes_primary_host_from_request_hostname(+ self,+ ) -> None:+ body = (+ b"host={{host}} "+ b"location={{location[hostname]}} "+ b"HTTP_ORIGIN: 'http://' + ORIGINAL_HOST + HTTP_PORT_ELIDED,"+ )++ self.assertEqual(+ _substitute_wpt_template_variables(+ body,+ port=12345,+ request_hostname="www1.localhost",+ primary_hostname="localhost",+ ),+ b"host=localhost location=www1.localhost "+ b"HTTP_ORIGIN: 'http://' + ORIGINAL_HOST + HTTP_PORT_ELIDED,",+ )+ def test_host_header_hostname_preserves_ipv6_brackets(self) -> None:+ self.assertEqual(_host_header_hostname("[2001:db8::1]:1234"), "[2001:db8::1]")+ self.assertEqual(_host_header_hostname("localhost:1234"), "localhost")+ self.assertEqual(_host_header_hostname("example.test"), "example.test")+ def test_fixture_server_substitutes_ipv6_websocket_template_urls(self) -> None:+ body = (+ b"ws=ws://{{host}}:{{ports[ws][0]}}/echo"+ b" wss=wss://{{host}}:{{ports[wss][0]}}/echo"+ )++ self.assertEqual(+ _substitute_wpt_template_variables(+ body,+ port=12345,+ request_hostname="[2001:db8::1]",+ ),+ b"ws=ws://[2001:db8::1]:12345/echo"+ b" wss=ws://[2001:db8::1]:12345/echo",+ )+ def test_fixture_server_wasm_status_handler_normalizes_status_codes(self) -> None:+ self.assertEqual(_wasm_webapi_status_code("status=404"), 404)+ self.assertEqual(_wasm_webapi_status_code("status=300"), 300)+ self.assertEqual(_wasm_webapi_status_code("status=0"), 599)+ self.assertEqual(_wasm_webapi_status_code("status=700"), 599)+ self.assertEqual(_wasm_webapi_status_code("status=not-a-number"), 400)+ self.assertEqual(_wasm_webapi_status_code(""), 200)+ def test_fixture_server_redirect_handler_models_wasm_origin_probe(self) -> None:+ self.assertEqual(+ _redirect_fixture_response(+ "redirect_status=301&location=/wasm/incrementer.wasm"+ ),+ (301, "/wasm/incrementer.wasm"),+ )+ self.assertIsNone(_redirect_fixture_response("redirect_status=200&location=/x"))+ self.assertIsNone(_redirect_fixture_response("redirect_status=301"))+ self.assertIsNone(+ _redirect_fixture_response("redirect_status=301&location=/x%0Dbad")+ )+ self.assertEqual(+ _redirect_fixture_response("status=307&location=/target"),+ (307, "/target"),+ )+ def test_fixture_server_models_csp_resource_py(self) -> None:+ body, headers = _content_security_policy_resource_response()++ self.assertIn(b"success", body)+ self.assertIn(("Access-Control-Allow-Origin", "*"), headers)+ def test_fixture_server_models_workers_modules_export_on_load_script_py(self) -> None:+ body, headers = _workers_modules_export_on_load_script_response()++ self.assertIn(b"export const importedModules", body)+ self.assertIn(("Content-Type", "text/javascript"), headers)+ self.assertIn(("Access-Control-Allow-Origin", "*"), headers)+ def test_fixture_server_preserves_raw_sidecar_header_bytes(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ resource = root / "resource.webmanifest"+ resource.write_bytes(b"{}")+ resource.with_name(resource.name + ".headers").write_bytes(+ b"Content-Type: \xc3\x97\xc2\xba invalid\n"+ )++ headers = _sidecar_response_headers(resource)++ self.assertEqual(headers, [("Content-Type", "\xc3\x97\xc2\xba invalid")])+ for _name, value in headers:+ value.encode("latin-1")+ def test_fixture_server_substitutes_get_host_info_remote_host_for_loopback(self) -> None:+ body = (+ b"var REMOTE_HOST = (ORIGINAL_HOST === 'localhost') ? "+ b"'127.0.0.1' : ('www1.' + ORIGINAL_HOST);"+ b"\nHTTP_REMOTE_ORIGIN: 'http://' + REMOTE_HOST + HTTP_PORT_ELIDED,"+ b"\nREMOTE_ORIGIN: PROTOCOL + \"//\" + REMOTE_HOST + PORT_ELIDED,"+ b"\nOTHER_ORIGIN: PROTOCOL + \"//\" + OTHER_HOST + PORT_ELIDED,"+ b"\nHTTP_NOTSAMESITE_ORIGIN: 'http://' + NOTSAMESITE_HOST + HTTP_PORT_ELIDED,"+ b"\nHTTPS_ORIGIN: 'https://' + ORIGINAL_HOST + HTTPS_PORT_ELIDED,"+ b"\nHTTPS_ORIGIN_WITH_CREDS: 'https://foo:bar@' + ORIGINAL_HOST + HTTPS_PORT_ELIDED,"+ b"\nHTTPS_REMOTE_ORIGIN: 'https://' + REMOTE_HOST + HTTPS_PORT_ELIDED,"+ b"\nHTTPS_REMOTE_ORIGIN_WITH_CREDS: 'https://foo:bar@' + REMOTE_HOST + HTTPS_PORT_ELIDED,"+ )++ self.assertEqual(+ _substitute_wpt_template_variables(+ body,+ port=12345,+ alternate_port=23456,+ remote_port=34567,+ ),+ b"var REMOTE_HOST = (ORIGINAL_HOST === 'localhost') ? "+ b"'www1.localhost' : ((ORIGINAL_HOST.indexOf(':') !== -1) ? "+ b"ORIGINAL_HOST : ('www1.' + ORIGINAL_HOST));"+ b"\nHTTP_REMOTE_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "+ b"('http://' + REMOTE_HOST + ':23456') : ('http://' + REMOTE_HOST + HTTP_PORT_ELIDED),"+ b"\nREMOTE_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "+ b"('http://' + REMOTE_HOST + ':23456') : (PROTOCOL + \"//\" + REMOTE_HOST + PORT_ELIDED),"+ b"\nOTHER_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "+ b"('http://' + ORIGINAL_HOST + ':34567') : (PROTOCOL + \"//\" + OTHER_HOST + PORT_ELIDED),"+ b"\nHTTP_NOTSAMESITE_ORIGIN: 'http://' + NOTSAMESITE_HOST + HTTP_PORT_ELIDED,"+ b"\nHTTPS_ORIGIN: 'http://' + ORIGINAL_HOST + HTTP_PORT2_ELIDED,"+ b"\nHTTPS_ORIGIN_WITH_CREDS: 'http://foo:bar@' + ORIGINAL_HOST + HTTP_PORT2_ELIDED,"+ b"\nHTTPS_REMOTE_ORIGIN: (ORIGINAL_HOST.indexOf(':') !== -1) ? "+ b"('http://' + REMOTE_HOST + ':34567') : ('http://' + REMOTE_HOST + HTTP_PORT2_ELIDED),"+ b"\nHTTPS_REMOTE_ORIGIN_WITH_CREDS: (ORIGINAL_HOST.indexOf(':') !== -1) ? "+ b"('http://foo:bar@' + REMOTE_HOST + ':34567') : "+ b"('http://foo:bar@' + REMOTE_HOST + HTTP_PORT2_ELIDED),",+ )diff --git a/moli-benchmark/tests/test_wpt_cross_fixture_responses.py b/moli-benchmark/tests/test_wpt_cross_fixture_responses.pynew file mode 100644index 000000000..abf2a0a1c--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_fixture_responses.py@@ -0,0 +1,415 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossFixtureResponsesTests(WptCrossTestCase):+ def test_fixture_server_parses_wpt_trickle_pipe_delay(self) -> None:+ self.assertEqual(_pipe_trickle_delay_seconds("pipe=trickle(d1)&cachebust=1"), 1.0)+ self.assertEqual(_pipe_trickle_delay_seconds("pipe=header(X,Y)|trickle(d2.5)"), 2.5)+ self.assertEqual(+ _pipe_trickle_delay_seconds("pipe=trickle(d3)&pipe=trickle(d1)"),+ 3.0,+ )+ self.assertEqual(_pipe_trickle_delay_seconds("pipe=trickle(d999)"), 10.0)+ self.assertEqual(_pipe_trickle_delay_seconds("notpipe=trickle(d1)"), 0.0)+ def test_fixture_server_parses_delay_handler_duration(self) -> None:+ self.assertEqual(_wpt_delay_seconds("ms=3000"), 3.0)+ self.assertEqual(_wpt_delay_seconds("ms=2.5"), 0.0025)+ self.assertEqual(_wpt_delay_seconds("ms=250&ms=750"), 0.25)+ self.assertEqual(_wpt_delay_seconds(""), 0.5)+ self.assertIsNone(_wpt_delay_seconds("ms=invalid"))+ self.assertIsNone(_wpt_delay_seconds("ms=-1"))+ self.assertIsNone(_wpt_delay_seconds("ms=nan"))+ def test_fixture_server_models_xhr_delay_py_methods(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ with patch(+ "moli_benchmark.wpt_cross.server.time.sleep"+ ) as sleep_mock:+ url = f"{server.base_url}/xhr/resources/delay.py?ms=250"+ responses = []+ for method in ("GET", "HEAD", "POST", "OPTIONS", "YO"):+ request = Request(+ url,+ data=b"upload" if method in {"POST", "YO"} else None,+ method=method,+ )+ with urlopen(request, timeout=2) as response:+ responses.append(+ (+ method,+ response.status,+ response.read(),+ response.headers["Content-Type"],+ response.headers["Access-Control-Allow-Origin"],+ response.headers["Access-Control-Allow-Methods"],+ )+ )++ self.assertEqual(+ [call.args for call in sleep_mock.call_args_list],+ [(0.25,)] * 5,+ )+ self.assertEqual(+ responses,+ [+ ("GET", 200, b"TEST_DELAY", "text/plain", "*", "YO"),+ ("HEAD", 200, b"", "text/plain", "*", "YO"),+ ("POST", 200, b"TEST_DELAY", "text/plain", "*", "YO"),+ ("OPTIONS", 200, b"TEST_DELAY", "text/plain", "*", "YO"),+ ("YO", 200, b"TEST_DELAY", "text/plain", "*", "YO"),+ ],+ )+ def test_fixture_server_models_delayed_module_script_handler(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ with patch(+ "moli_benchmark.wpt_cross.server.time.sleep"+ ) as sleep_mock:+ url = (+ f"{server.base_url}/html/semantics/scripting-1/"+ "the-script-element/module/resources/"+ "delayed-modulescript.py?ms=250"+ )+ responses = []+ for method in ("GET", "HEAD"):+ request = Request(url, method=method)+ with urlopen(request, timeout=2) as response:+ responses.append(+ (+ method,+ response.status,+ response.read(),+ response.headers["Content-Type"],+ )+ )++ self.assertEqual(+ [call.args for call in sleep_mock.call_args_list],+ [(0.25,), (0.25,)],+ )+ self.assertEqual(+ responses,+ [+ (+ "GET",+ 200,+ b"export let delayedLoaded = true;",+ "text/javascript",+ ),+ ("HEAD", 200, b"", "text/javascript"),+ ],+ )+ def test_fixture_server_models_common_redirect_opt_in_handler(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ connection.request(+ "GET",+ "/common/redirect-opt-in.py?status=307&location=%2Ftarget",+ )+ response = connection.getresponse()+ self.assertEqual(response.status, 307)+ self.assertEqual(response.headers.get("Location"), "/target")+ self.assertEqual(response.headers.get("Timing-Allow-Origin"), "*")+ connection.close()+ def test_fixture_server_drains_xhr_delay_yo_body_before_next_request(+ self,+ ) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ server = WptFixtureServer(root_path)+ server.httpd.RequestHandlerClass.protocol_version = "HTTP/1.1"+ with server, patch(+ "moli_benchmark.wpt_cross.server.time.sleep"+ ):+ connection = HTTPConnection("localhost", server.port, timeout=2)+ try:+ path = "/xhr/resources/delay.py?ms=0"+ connection.request("YO", path, body=b"upload")+ first = connection.getresponse()+ self.assertEqual(+ (first.status, first.read()),+ (200, b"TEST_DELAY"),+ )+ first_socket = connection.sock++ connection.request("GET", path)+ second = connection.getresponse()+ self.assertEqual(+ (second.status, second.read()),+ (200, b"TEST_DELAY"),+ )+ self.assertIs(connection.sock, first_socket)+ finally:+ connection.close()+ def test_fixture_server_drains_chunked_xhr_delay_body_before_next_request(+ self,+ ) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ server = WptFixtureServer(root_path)+ server.httpd.RequestHandlerClass.protocol_version = "HTTP/1.1"+ with server, patch(+ "moli_benchmark.wpt_cross.server.time.sleep"+ ):+ connection = HTTPConnection("localhost", server.port, timeout=2)+ try:+ path = "/xhr/resources/delay.py?ms=0"+ connection.request(+ "POST",+ path,+ body=[b"chunk-one", b"chunk-two"],+ encode_chunked=True,+ )+ first = connection.getresponse()+ self.assertEqual(+ (first.status, first.read()),+ (200, b"TEST_DELAY"),+ )+ first_socket = connection.sock++ connection.request("GET", path)+ second = connection.getresponse()+ self.assertEqual(+ (second.status, second.read()),+ (200, b"TEST_DELAY"),+ )+ self.assertIs(connection.sock, first_socket)+ finally:+ connection.close()+ def test_fixture_server_parses_wpt_header_pipe(self) -> None:+ def applied_pipe_headers(query: str) -> list[tuple[str, str]]:+ return _apply_header_operations(+ [], _pipe_response_header_operations(query)+ )++ self.assertEqual(+ applied_pipe_headers(+ "pipe=header(Access-Control-Allow-Origin,*)|header(X-Test,ok)"+ ),+ [("Access-Control-Allow-Origin", "*"), ("X-Test", "ok")],+ )+ self.assertEqual(+ applied_pipe_headers(+ "pipe=header(Content-Security-Policy,first)"+ "|header(Content-Security-Policy,second,True)"+ ),+ [+ ("Content-Security-Policy", "first"),+ ("Content-Security-Policy", "second"),+ ],+ )+ self.assertEqual(+ applied_pipe_headers(+ "pipe=header(Content-Security-Policy,first)"+ "|header(Content-Security-Policy,second,False)"+ ),+ [("Content-Security-Policy", "second")],+ )+ self.assertEqual(applied_pipe_headers("pipe=header(Bad Header,ok)"), [])+ self.assertEqual(+ applied_pipe_headers("pipe=header(X-Test,bad%0D%0AInjected:%20x)"),+ [("X-Test", "bad Injected: x")],+ )+ self.assertEqual(applied_pipe_headers("notpipe=header(X,Y)"), [])+ def test_fixture_server_parses_wpt_status_pipe(self) -> None:+ self.assertEqual(_pipe_response_status("pipe=status(204)&cachebust=1"), 204)+ self.assertEqual(+ _pipe_response_status("pipe=header(X,Y)|status(205)"),+ 205,+ )+ self.assertEqual(+ _pipe_response_status("pipe=status(204)&pipe=status(205)"),+ 205,+ )+ self.assertIsNone(_pipe_response_status("pipe=status(099)"))+ self.assertIsNone(_pipe_response_status("pipe=status(600)"))+ self.assertIsNone(_pipe_response_status("notpipe=status(204)"))+ def test_fixture_server_parses_wpt_headers_sidecar(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ fixture = Path(root) / "case.html"+ fixture.write_text("<!doctype html>", encoding="utf-8")+ fixture.with_name("case.html.headers").write_text(+ "Referrer-Policy: no-referrer\n"+ "Bad Header: skip\n"+ "X-Test: ok\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _sidecar_response_headers(fixture),+ [("Referrer-Policy", "no-referrer"), ("X-Test", "ok")],+ )+ def test_fixture_server_parses_sub_headers_sidecar_for_sub_files(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ fixture = Path(root) / "worker.sub.js"+ fixture.write_text("// worker", encoding="utf-8")+ fixture.with_name("worker.sub.js.sub.headers").write_text(+ "Content-Security-Policy: connect-src 'none'\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _sidecar_response_headers(fixture),+ [("Content-Security-Policy", "connect-src 'none'")],+ )+ def test_fixture_server_combines_immediate_directory_and_file_headers(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ fixture_dir = root_path / "support"+ fixture_dir.mkdir()+ fixture = fixture_dir / "ufoo"+ fixture.write_text("ufoo", encoding="utf-8")+ fixture_dir.joinpath("__dir__.headers").write_text(+ "Content-Type: text/html\nX-Directory: immediate\n",+ encoding="utf-8",+ )+ fixture.with_name("ufoo.headers").write_text(+ "X-File: exact\n",+ encoding="utf-8",+ )+ root_path.joinpath("__dir__.headers").write_text(+ "X-Directory: parent\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _sidecar_response_headers(fixture),+ [+ ("Content-Type", "text/html"),+ ("X-Directory", "immediate"),+ ("X-File", "exact"),+ ],+ )+ def test_fixture_server_prefers_file_sub_headers_when_both_exist(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ fixture = Path(root) / "case.sub.html"+ fixture.write_text("<!doctype html>", encoding="utf-8")+ fixture.with_name("case.sub.html.headers").write_text(+ "X-Source: plain\n",+ encoding="utf-8",+ )+ fixture.with_name("case.sub.html.sub.headers").write_text(+ "X-Source: substituted\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _sidecar_response_headers(fixture),+ [("X-Source", "substituted")],+ )+ def test_fixture_server_prefers_directory_sub_headers_when_both_exist(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ fixture = root_path / "case.html"+ fixture.write_text("<!doctype html>", encoding="utf-8")+ root_path.joinpath("__dir__.headers").write_text(+ "X-Source: plain\n",+ encoding="utf-8",+ )+ root_path.joinpath("__dir__.sub.headers").write_text(+ "X-Source: substituted\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _sidecar_response_headers(fixture),+ [("X-Source", "substituted")],+ )+ def test_fixture_server_content_type_sidecar_overrides_static_guess(self) -> None:+ content_type, headers = _response_content_type_and_extra_headers(+ "application/octet-stream",+ [+ ("Content-Type", "text/javascript; charset=utf-8"),+ ("X-Test", "ok"),+ ],+ )++ self.assertEqual(content_type, "text/javascript; charset=utf-8")+ self.assertEqual(headers, [("X-Test", "ok")])+ def test_fixture_server_combines_sidecar_and_pipe_headers_for_static_responses(+ self,+ ) -> None:+ with tempfile.TemporaryDirectory() as root:+ fixture = Path(root) / "feature.any.js"+ fixture.write_text("// test", encoding="utf-8")+ fixture.with_name("feature.any.js.headers").write_text(+ "Content-Security-Policy: script-src 'self'\n",+ encoding="utf-8",+ )++ self.assertEqual(+ _static_response_headers(fixture, "pipe=header(X-Test,ok)"),+ [+ ("Content-Security-Policy", "script-src 'self'"),+ ("X-Test", "ok"),+ ],+ )+ self.assertEqual(+ _static_response_headers(+ fixture,+ "pipe=header(Content-Security-Policy,script-src 'none',False)",+ ),+ [("Content-Security-Policy", "script-src 'none'")],+ )+ self.assertEqual(+ _static_response_headers(+ fixture,+ "pipe=header(Content-Security-Policy,require-trusted-types-for 'script',True)",+ ),+ [+ ("Content-Security-Policy", "script-src 'self'"),+ (+ "Content-Security-Policy",+ "require-trusted-types-for 'script'",+ ),+ ],+ )+ def test_fixture_server_reads_sub_headers_for_plain_static_resource(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ fixture = Path(root) / "policy.html"+ fixture.write_text("<!doctype html>", encoding="utf-8")+ fixture.with_name("policy.html.sub.headers").write_text(+ "Set-Cookie: policy={{$id:uuid()}}\n"+ "Content-Security-Policy: frame-src 'none'; report-uri /report.py?reportID={{$id}}\n",+ encoding="utf-8",+ )++ headers = _static_response_headers(fixture, "", port=8000)++ self.assertEqual(len(headers), 2)+ cookie_value = headers[0][1].removeprefix("policy=")+ self.assertEqual(headers[0][0], "Set-Cookie")+ self.assertNotIn("{{", cookie_value)+ self.assertEqual(+ headers[1],+ (+ "Content-Security-Policy",+ f"frame-src 'none'; report-uri /report.py?reportID={cookie_value}",+ ),+ )diff --git a/moli-benchmark/tests/test_wpt_cross_html_reports.py b/moli-benchmark/tests/test_wpt_cross_html_reports.pynew file mode 100644index 000000000..cd1e118d7--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_html_reports.py@@ -0,0 +1,157 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossHtmlReportsTests(WptCrossTestCase):+ def test_render_html_escapes_embedded_json_script_data(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir)+ (output_dir / "summary.json").write_text(+ json.dumps({"total": 1, "engines": {"moli": {"pass": 1}}}),+ encoding="utf-8",+ )+ (output_dir / "matrix.json").write_text(+ json.dumps(+ [+ {+ "case_path": "xss.html",+ "results": {+ "moli": {+ "status": "</script><img src=x onerror=alert(1)>",+ "duration_ms": 1,+ }+ },+ }+ ]+ ),+ encoding="utf-8",+ )++ html = render_html(output_dir).read_text(encoding="utf-8")++ self.assertNotIn("</script><img", html)+ self.assertIn("\\u003c/script\\u003e", html)+ def test_render_html_includes_recorded_failure_drift(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir)+ (output_dir / "summary.json").write_text(+ json.dumps(+ {+ "total": 1,+ "engines": {"moli": {"fail": 1}, "chrome": {"fail": 1}},+ "recorded_failure_drift": {+ "primary": "moli",+ "recorded_failure_limit_per_engine": 40,+ "comparison_count": 1,+ "comparisons": [+ {+ "case_path": "WebCryptoAPI/shared.html",+ "primary": "moli",+ "peer": "chrome",+ "primary_only_count": 1,+ "peer_only_count": 0,+ "message_diff_count": 0,+ "primary_only_examples": ["lm-only"],+ "peer_only_examples": [],+ "message_diff_examples": [],+ }+ ],+ },+ }+ ),+ encoding="utf-8",+ )+ (output_dir / "matrix.json").write_text(+ json.dumps(+ [+ {+ "case_path": "WebCryptoAPI/shared.html",+ "results": {+ "moli": {"status": "fail", "duration_ms": 1},+ "chrome": {"status": "fail", "duration_ms": 1},+ },+ }+ ]+ ),+ encoding="utf-8",+ )++ html = render_html(output_dir).read_text(encoding="utf-8")++ self.assertIn("Recorded subtest drift", html)+ self.assertIn("lm-only", html)+ def test_render_html_includes_known_failure_audit_summary(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir)+ (output_dir / "summary.json").write_text(+ json.dumps(+ {+ "total": 1,+ "engines": {"moli": {"fail": 1}},+ "known_failure_audits": {+ "moli": {+ "ok": False,+ "manifest": "known.json",+ "output": "known-failure-audit-moli.json",+ "counts": {+ "known_failures": 1,+ "resolved_known_failures": 1,+ "mismatched_known_failures": 0,+ "missing_expected_failures": 0,+ "skipped_known_failures": 2,+ "unexpected_failures": 0,+ },+ "category_counts": {+ "known_failures": {+ "wasm-global-live-binding": 1,+ }+ },+ "categories": {+ "wasm-global-live-binding": {+ "tracking_doc": "docs/wasm-global-live-binding-design-current.md",+ "scope": "V8-backed live binding work",+ "evidence": [+ {+ "kind": "doc",+ "path": "docs/wasm-global-live-binding-design-current.md",+ "note": "local fixture evidence",+ }+ ],+ }+ },+ }+ },+ }+ ),+ encoding="utf-8",+ )+ (output_dir / "matrix.json").write_text(+ json.dumps(+ [+ {+ "case_path": "wasm/example.html",+ "results": {+ "moli": {+ "status": "fail",+ "duration_ms": 1,+ }+ },+ }+ ]+ ),+ encoding="utf-8",+ )++ html = render_html(output_dir).read_text(encoding="utf-8")++ self.assertIn("Known-failure audit", html)+ self.assertIn("known-failure-audit-moli.json", html)+ self.assertIn("wasm-global-live-binding", html)+ self.assertIn("docs/wasm-global-live-binding-design-current.md", html)+ self.assertIn("V8-backed live binding work", html)+ self.assertIn("doc <code>docs/wasm-global-live-binding-design-current.md</code>", html)+ self.assertIn("local fixture evidence", html)+ self.assertIn("<th>skipped</th>", html)+ self.assertIn("<td>2</td>", html)+ self.assertIn(">attention<", html)diff --git a/moli-benchmark/tests/test_wpt_cross_result_reporting.py b/moli-benchmark/tests/test_wpt_cross_result_reporting.pynew file mode 100644index 000000000..ab2a7eee3--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_result_reporting.py@@ -0,0 +1,397 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossResultReportingTests(WptCrossTestCase):+ def test_report_bridge_keys_results_by_initial_search_without_hash(self) -> None:+ self.assertIn(b"var initialCasePath =", BENCH_REPORT_BRIDGE)+ self.assertIn(b"case_path: initialCasePath", BENCH_REPORT_BRIDGE)+ self.assertIn(b"location.pathname + location.search", BENCH_REPORT_BRIDGE)+ self.assertNotIn(b"location.pathname + location.search + location.hash", BENCH_REPORT_BRIDGE)+ def test_report_bridge_does_not_post_incremental_payloads(self) -> None:+ self.assertIn(b"if (source === 'incremental') {", BENCH_REPORT_BRIDGE)+ incremental_start = BENCH_REPORT_BRIDGE.index(b"if (source === 'incremental') {")+ final_payload_start = BENCH_REPORT_BRIDGE.index(b"var snapshot = {")+ incremental_block = BENCH_REPORT_BRIDGE[incremental_start:final_payload_start]++ self.assertIn(b"partial_count: accumulator.tests.length", incremental_block)+ self.assertIn(b"tests: []", incremental_block)+ self.assertIn(b"return;", incremental_block)+ self.assertNotIn(b"accumulator.tests.slice()", incremental_block)+ def test_report_bridge_posts_only_bounded_final_payloads_synchronously(self) -> None:+ self.assertIn(b"if (body.length <= 60000)", BENCH_REPORT_BRIDGE)+ self.assertIn(b"xhr.open('POST', '/__bench__/result', false)", BENCH_REPORT_BRIDGE)+ self.assertLess(+ BENCH_REPORT_BRIDGE.index(b"xhr.open('POST', '/__bench__/result', false)"),+ BENCH_REPORT_BRIDGE.rindex(b"window.__bench_wpt__ = snapshot;"),+ )+ def test_report_bridge_writes_dom_payload_before_posting(self) -> None:+ self.assertIn(b"__bench_wpt_payload", BENCH_REPORT_BRIDGE)+ self.assertLess(+ BENCH_REPORT_BRIDGE.index(b"node.textContent = body;"),+ BENCH_REPORT_BRIDGE.index(b"xhr.open('POST', '/__bench__/result', false)"),+ )+ def test_report_bridge_disables_in_page_testharness_output(self) -> None:+ self.assertIn(b"output: false", BENCH_REPORT_BRIDGE)+ def test_report_bridge_applies_configured_timeout_multiplier(self) -> None:+ bridge = _bench_report_bridge(7.5)++ self.assertIn(b"timeout_multiplier: 7.5", bridge)+ def test_report_bridge_config_is_injected_without_changing_case_url(self) -> None:+ body = (+ b'<!doctype html><script src="/resources/testharness.js"></script>'+ b'<script src="/resources/testharnessreport.js"></script>'+ b"<script>test(() => location.search, 'query visible to case')</script>"+ )++ injected = _inject_bench_report_bridge_config(body, 3.0)++ self.assertIn(+ (+ b'src="/resources/testharnessreport.js?'+ + BENCH_TIMEOUT_MULTIPLIER_QUERY.encode("ascii")+ + b'=3"'+ ),+ injected,+ )+ self.assertIn(b"location.search", injected)+ def test_report_bridge_config_keeps_default_multiplier_body_unchanged(self) -> None:+ body = (+ b'<!doctype html><script src="/resources/testharness.js"></script>'+ b'<script src="/resources/testharnessreport.js"></script>'+ )++ self.assertEqual(_inject_bench_report_bridge_config(body, 1.0), body)+ def test_harness_case_key_ignores_url_fragment(self) -> None:+ self.assertEqual(+ _normalize_harness_case_key("/dom/ranges/feature.html?mode=open#frag"),+ "dom/ranges/feature.html?mode=open",+ )+ def test_report_bridge_does_not_publish_empty_done_fallback(self) -> None:+ self.assertIn(b"function canPublishDoneFallback()", BENCH_REPORT_BRIDGE)+ self.assertIn(+ b"window.__bench_wpt__ === undefined && canPublishDoneFallback()",+ BENCH_REPORT_BRIDGE,+ )+ self.assertIn(+ b"canPublishDoneFallback()) {\n publish('done-hook-late', null);",+ BENCH_REPORT_BRIDGE,+ )+ def test_case_result_dict_records_payload_source(self) -> None:+ row = case_result_to_dict(+ CaseResult(+ case_path="example.html",+ url="http://example.test/example.html",+ status="pass",+ duration_ms=1.0,+ payload_source="completion-callback",+ )+ )++ self.assertEqual(row["payload_source"], "completion-callback")+ def test_case_result_dict_records_reftest_comparisons_and_artifacts(self) -> None:+ row = case_result_to_dict(+ CaseResult(+ case_path="css/example.html",+ url="http://example.test/css/example.html",+ status="fail",+ duration_ms=1.0,+ test_type="reftest",+ reftest_comparisons=[+ {+ "reference_path": "css/example-ref.html",+ "relation": "==",+ "passed": False,+ "max_difference": 255,+ "different_pixels": 10,+ }+ ],+ artifacts={+ "test": "artifacts/moli/example/test.png",+ "references": [+ {+ "reference": "artifacts/moli/example/reference-01.png",+ "diff": "artifacts/moli/example/diff-01.png",+ }+ ],+ },+ )+ )++ self.assertEqual(row["test_type"], "reftest")+ self.assertEqual(row["reftest_comparisons"][0]["relation"], "==")+ self.assertEqual(row["artifacts"]["test"], "artifacts/moli/example/test.png")+ def test_cli_runner_extracts_payload_from_stdout_html(self) -> None:+ payload = {+ "case_path": "/case.html",+ "harness": {"status": 0, "message": None},+ "tests": [{"name": "ok", "status": 0}],+ "source": "completion-callback",+ }+ stdout = (+ "<!doctype html><html><pre id=\"__bench_wpt_payload\" hidden>"+ + json.dumps(payload)+ + "</pre></html>"+ ).encode()++ self.assertEqual(_payload_from_stdout_html(stdout), payload)+ def test_cli_runner_prefers_final_stdout_payload_without_callback_wait(self) -> None:+ class UnexpectedResultsAccess:+ def wait_for_final(self, key: str, timeout: float) -> None:+ raise AssertionError("final stdout payload should skip callback wait")++ def get(self, key: str) -> None:+ raise AssertionError("final stdout payload should skip callback lookup")++ payload = {+ "case_path": "/case.html",+ "harness": {"status": 0, "message": None},+ "tests": [{"name": "ok", "status": 0}],+ "source": "completion-callback",+ }+ stdout = (+ '<!doctype html><pre id="__bench_wpt_payload" hidden>'+ + json.dumps(payload)+ + "</pre>"+ ).encode()++ result = _classify_cli_case_result(+ case_path="case.html",+ url="http://example.test/case.html",+ bridge_key="/case.html",+ fixture_server=SimpleNamespace(results=UnexpectedResultsAccess()),+ subprocess_result=_CliSubprocessResult(+ duration_ms=10.0,+ proc_error=None,+ proc_returncode=0,+ proc_stderr="",+ proc_stdout=stdout,+ wait_script_timeout=False,+ ),+ payload_grace_seconds=2.0,+ successful_process_payload_grace_seconds=8.0,+ )++ self.assertEqual(result.status, "pass")+ self.assertEqual(result.payload_source, "completion-callback")+ def test_case_result_dict_records_failure_details(self) -> None:+ row = case_result_to_dict(+ CaseResult(+ case_path="example.html",+ url="http://example.test/example.html",+ status="fail",+ duration_ms=1.0,+ failures=[+ {+ "name": "subtest name",+ "status": 1,+ "status_name": "FAIL",+ "message": "expected true got false",+ }+ ],+ )+ )++ self.assertEqual(+ row["failures"],+ [+ {+ "name": "subtest name",+ "status": 1,+ "status_name": "FAIL",+ "message": "expected true got false",+ }+ ],+ )+ def test_classify_payload_records_limited_failure_details(self) -> None:+ result = classify_payload(+ payload={+ "source": "completion-callback",+ "harness": {"status": 0},+ "tests": [+ {"name": "ok", "status": 0},+ {+ "name": "bad",+ "status": 1,+ "message": "x" * 700,+ },+ {"name": "notrun", "status": 3},+ ],+ },+ case_path="example.html",+ url="http://example.test/example.html",+ duration_ms=1.0,+ bridge_installed=True,+ )++ self.assertEqual(result.status, "fail")+ self.assertEqual(result.subtests_total, 3)+ self.assertEqual([failure["name"] for failure in result.failures], ["bad", "notrun"])+ self.assertEqual(result.failure_names, ["bad", "notrun"])+ self.assertEqual(result.failures[0]["status_name"], "FAIL")+ self.assertEqual(len(result.failures[0]["message"]), 500)+ self.assertTrue(result.failures[0]["message_truncated"])+ def test_recorded_failure_drift_reports_hidden_subtest_differences(self) -> None:+ drift = _recorded_failure_drift(+ [+ {+ "case_path": "webcrypto/shared-fail.html",+ "results": {+ "moli": {+ "status": "fail",+ "failures": [+ {"name": "shared", "message": "lm detail"},+ {"name": "lm-only", "message": "bad"},+ ],+ },+ "chrome": {+ "status": "fail",+ "failures": [+ {"name": "shared", "message": "chrome detail"},+ {"name": "chrome-only", "message": "bad"},+ ],+ },+ },+ }+ ],+ ["moli", "chrome"],+ )++ self.assertEqual(drift["comparison_count"], 1)+ self.assertEqual(drift["primary_only_comparison_count"], 1)+ self.assertEqual(drift["peer_only_comparison_count"], 1)+ self.assertEqual(drift["message_diff_comparison_count"], 1)+ row = drift["comparisons"][0]+ self.assertEqual(row["primary_only_examples"], ["lm-only"])+ self.assertEqual(row["peer_only_examples"], ["chrome-only"])+ self.assertEqual(row["message_diff_examples"], ["shared"])+ def test_recorded_failure_drift_uses_full_failure_names_when_available(self) -> None:+ drift = _recorded_failure_drift(+ [+ {+ "case_path": "webcrypto/large-shared-fail.html",+ "results": {+ "moli": {+ "status": "fail",+ "failures": [{"name": "recorded", "message": "same"}],+ "failure_names": ["recorded", "late-moli-only"],+ },+ "chrome": {+ "status": "fail",+ "failures": [{"name": "recorded", "message": "same"}],+ "failure_names": ["recorded"],+ },+ },+ }+ ],+ ["moli", "chrome"],+ )++ self.assertEqual(drift["comparison_count"], 1)+ self.assertEqual(drift["primary_only_comparison_count"], 1)+ self.assertEqual(drift["comparisons"][0]["primary_only_examples"], ["late-moli-only"])+ def test_classify_payload_does_not_pass_incremental_only_result(self) -> None:+ result = classify_payload(+ payload={+ "source": "incremental",+ "harness": {"status": None},+ "tests": [{"name": "observed pass", "status": 0}],+ },+ case_path="example.html",+ url="http://example.test/example.html",+ duration_ms=1.0,+ bridge_installed=True,+ )++ self.assertEqual(result.status, "harness-stalled")+ self.assertEqual(result.subtests_total, 1)+ self.assertEqual(result.subtests_pass, 1)+ self.assertEqual(result.payload_source, "incremental")+ self.assertIn("incremental", result.error or "")+ def test_classify_payload_does_not_pass_empty_final_result(self) -> None:+ result = classify_payload(+ payload={+ "source": "completion-callback",+ "harness": {"status": 0},+ "tests": [],+ },+ case_path="example.html",+ url="http://example.test/example.html",+ duration_ms=1.0,+ bridge_installed=True,+ )++ self.assertEqual(result.status, "fail")+ self.assertEqual(result.subtests_total, 0)+ self.assertIn("without reporting any subtests", result.error or "")+ def test_classify_payload_includes_empty_harness_error_message(self) -> None:+ result = classify_payload(+ payload={+ "source": "completion-callback",+ "harness": {+ "status": 1,+ "message": "Unhandled rejection: cyclic wasm dependency",+ },+ "tests": [],+ },+ case_path="example.html",+ url="http://example.test/example.html",+ duration_ms=1.0,+ bridge_installed=True,+ )++ self.assertEqual(result.status, "fail")+ self.assertEqual(result.harness_status, 1)+ self.assertEqual(result.subtests_total, 0)+ self.assertIn("without reporting any subtests", result.error or "")+ self.assertIn("cyclic wasm dependency", result.error or "")+ def test_classify_payload_keeps_observed_incremental_failure(self) -> None:+ result = classify_payload(+ payload={+ "source": "incremental",+ "harness": {"status": None},+ "tests": [{"name": "observed fail", "status": 1}],+ },+ case_path="example.html",+ url="http://example.test/example.html",+ duration_ms=1.0,+ bridge_installed=True,+ )++ self.assertEqual(result.status, "fail")+ self.assertEqual(result.subtests_fail, 1)+ self.assertEqual(result.payload_source, "incremental")+ def test_results_store_wait_for_final_does_not_return_incremental(self) -> None:+ store = ResultsStore()+ store.put("example.html", {"source": "incremental"})++ self.assertIsNone(store.wait_for_final("example.html", timeout=0))+ self.assertEqual(store.get("example.html"), {"source": "incremental"})+ def test_testdriver_vendor_bridge_provides_action_sequence(self) -> None:+ self.assertIn(b"action_sequence", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"pointerMove", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ def test_testdriver_vendor_bridge_preserves_key_modifiers(self) -> None:+ self.assertIn(b"altKey: !!modifiers.Alt", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"ctrlKey: !!modifiers.Control", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"shiftKey: !!modifiers.Shift", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ def test_testdriver_vendor_bridge_focuses_user_activation_target(self) -> None:+ self.assertIn(b"focusForUserActivation", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"document.activeElement === before", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ def test_testdriver_vendor_bridge_falls_back_to_recorded_hit_test_targets(self) -> None:+ self.assertIn(b"nativeGetClientRects.apply", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"nativeElementsFromPoint.apply", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"return target ? [target] : []", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ def test_testdriver_vendor_bridge_accepts_storage_access_permission_setup(self) -> None:+ self.assertIn(+ b"params.descriptor.name === 'storage-access'",+ BENCH_TESTDRIVER_VENDOR_BRIDGE,+ )+ self.assertIn(+ b"set_permission() is not implemented by the Moli WPT bridge",+ BENCH_TESTDRIVER_VENDOR_BRIDGE,+ )+ def test_testdriver_vendor_bridge_provides_computed_label(self) -> None:+ self.assertIn(b"get_computed_label", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"resolveReferenceTarget", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"data-expectedlabel", BENCH_TESTDRIVER_VENDOR_BRIDGE)diff --git a/moli-benchmark/tests/test_wpt_cross_runner_orchestration.py b/moli-benchmark/tests/test_wpt_cross_runner_orchestration.pynew file mode 100644index 000000000..ed7f3bd4c--- /dev/null+++ b/moli-benchmark/tests/test_wpt_cross_runner_orchestration.py@@ -0,0 +1,782 @@+from __future__ import annotations++from wpt_cross_test_support import *+++class WptCrossRunnerOrchestrationTests(WptCrossTestCase):+ def test_main_writes_schedule_and_passes_execution_order_to_cli_runner(self) -> None:+ captured: dict[str, object] = {}++ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.base_url = "http://127.0.0.1:8000"+ self.alternate_base_url = "http://127.0.0.1:8001"+ self.external_base_url = None+ self.external_alternate_base_url = None+ self.external_remote_base_url = None+ self.external_host = None++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ return None++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ return f"{self.base_url}/{case_path}"++ cases = [+ WptCase("content-security-policy/navigation/a.html"),+ WptCase("content-security-policy/navigation/b.html"),+ WptCase("html/browsers/a.html"),+ WptCase("html/browsers/b.html"),+ WptCase("trusted-types/reporting/a.html"),+ WptCase("trusted-types/reporting/b.html"),+ ]+ scheduled_cases, expected_metadata = build_run_schedule(+ cases,+ case_path=lambda case: case.case_path,+ )+ result_dict = {+ "engine": "moli",+ "binary": "/tmp/moli",+ "binary_sha256": "sha",+ "binary_version": "0.1.0",+ "endpoint": "cli:/tmp/moli",+ "ready_ms": None,+ "setup_error": None,+ "cases": [],+ }++ def fake_run_engine_on_cases_cli(**kwargs: object) -> SimpleNamespace:+ captured["cases"] = kwargs["cases"]+ captured["execution_cases"] = kwargs["execution_cases"]+ captured["case_timeout_seconds"] = kwargs["case_timeout_seconds"]+ captured["parallelism"] = kwargs["parallelism"]+ return SimpleNamespace(setup_error=None)++ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir) / "out"+ with (+ patch(+ "moli_benchmark.wpt_cross.__main__.enumerate_cases",+ return_value=cases,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.build_driver",+ return_value=SimpleNamespace(cli_fetch_command=["moli"]),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",+ side_effect=fake_run_engine_on_cases_cli,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",+ return_value=result_dict,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",+ Path(temp_dir) / "repo-case-lists",+ ),+ patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),+ redirect_stdout(StringIO()),+ redirect_stderr(StringIO()),+ ):+ code = main(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ str(output_dir),+ ]+ )+ self.assertEqual(code, 0)+ self.assertEqual(+ (output_dir / "cases.txt").read_text(encoding="utf-8").splitlines(),+ [case.case_path for case in cases],+ )+ self.assertEqual(+ (output_dir / "schedule.txt").read_text(encoding="utf-8").splitlines(),+ [case.case_path for case in scheduled_cases],+ )+ schedule_json = json.loads(+ (output_dir / "schedule.json").read_text(encoding="utf-8")+ )+ self.assertEqual(schedule_json, expected_metadata)+ self.assertEqual(+ schedule_json["mode"],+ "fixed-prefix-balanced-shuffle",+ )+ self.assertEqual(+ [case[0] for case in captured["cases"]],+ [case.case_path for case in cases],+ )+ self.assertEqual(+ [case[0] for case in captured["execution_cases"]],+ [case.case_path for case in scheduled_cases],+ )+ self.assertEqual(captured["case_timeout_seconds"], WPT_CROSS_CASE_TIMEOUT_SECONDS)+ self.assertEqual(captured["parallelism"], WPT_CROSS_PARALLELISM)+ def test_main_uses_requested_parallelism_for_cdp_runner(self) -> None:+ calls: list[dict[str, object]] = []++ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.base_url = "http://127.0.0.1:8000"+ self.alternate_base_url = "http://127.0.0.1:8001"+ self.external_base_url = None+ self.external_alternate_base_url = None+ self.external_remote_base_url = None+ self.external_host = None++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ return None++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ return f"{self.base_url}/{case_path}"++ def fake_run_engine_on_cases(**kwargs: object) -> EngineRunResult:+ cases_arg = kwargs["cases"]+ assert isinstance(cases_arg, list)+ calls.append(+ {+ "cases": cases_arg,+ "case_timeout_seconds": kwargs["case_timeout_seconds"],+ }+ )+ case_path, url, _timeout = cases_arg[0]+ return EngineRunResult(+ engine="chrome",+ binary="/tmp/chrome",+ binary_sha256="sha",+ binary_version="version",+ endpoint="cdp://127.0.0.1:1",+ ready_ms=1.0,+ cases=[CaseResult(case_path, url, "pass", 1.0)],+ )++ result_dict = {+ "engine": "chrome",+ "binary": "/tmp/chrome",+ "binary_sha256": "sha",+ "binary_version": "version",+ "endpoint": "cdp://127.0.0.1:1",+ "ready_ms": 1.0,+ "setup_error": None,+ "cases": [],+ }++ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir) / "out"+ cases = [WptCase(f"case-{index}.html") for index in range(3)]+ with (+ patch(+ "moli_benchmark.wpt_cross.__main__.enumerate_cases",+ return_value=cases,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.build_driver",+ return_value=SimpleNamespace(cli_fetch_command=None),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases",+ side_effect=fake_run_engine_on_cases,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",+ return_value=result_dict,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",+ Path(temp_dir) / "repo-case-lists",+ ),+ patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),+ redirect_stdout(StringIO()),+ redirect_stderr(StringIO()),+ ):+ code = main(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "chrome",+ "--output-dir",+ str(output_dir),+ "--mode",+ "cdp",+ "--parallelism",+ "2",+ ]+ )++ self.assertEqual(code, 0)+ self.assertEqual(len(calls), 2)+ self.assertTrue(all(call["case_timeout_seconds"] == WPT_CROSS_CASE_TIMEOUT_SECONDS for call in calls))+ self.assertEqual(+ sorted(case[0] for call in calls for case in call["cases"]),+ [case.case_path for case in cases],+ )+ def test_layout_testharness_profile_forces_cdp_and_fixed_viewport(self) -> None:+ captured: dict[str, object] = {}++ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.base_url = "http://127.0.0.1:8000"+ self.alternate_base_url = "http://127.0.0.1:8001"+ self.external_base_url = None+ self.external_alternate_base_url = None+ self.external_remote_base_url = None+ self.external_host = None++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ return None++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ return f"{self.base_url}/{case_path}"++ def fake_cdp_run(**kwargs: object) -> EngineRunResult:+ captured.update(kwargs)+ cases_arg = kwargs["cases"]+ assert isinstance(cases_arg, list)+ case_path, url, _timeout = cases_arg[0]+ return EngineRunResult(+ engine="moli",+ binary="/tmp/moli",+ binary_sha256="sha",+ binary_version="version",+ endpoint="cdp://127.0.0.1:1",+ ready_ms=1.0,+ cases=[CaseResult(case_path, url, "pass", 1.0)],+ )++ case = WptCase("css/css-flexbox/layout.html")+ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir) / "out"+ with (+ patch(+ "moli_benchmark.wpt_cross.__main__.enumerate_cases",+ return_value=[case],+ ) as enumerate_mock,+ patch(+ "moli_benchmark.wpt_cross.__main__.build_driver",+ return_value=SimpleNamespace(cli_fetch_command=lambda *_: []),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases",+ side_effect=fake_cdp_run,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",+ side_effect=AssertionError("layout profile must not use CLI mode"),+ ),+ patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),+ redirect_stdout(StringIO()),+ redirect_stderr(StringIO()),+ ):+ code = main(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ str(output_dir),+ "--profile",+ "layout-testharness",+ ]+ )++ summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))++ self.assertEqual(code, 0)+ self.assertEqual(captured["viewport"], LAYOUT_VIEWPORT)+ self.assertIsNone(captured["artifact_output_dir"])+ self.assertTrue(enumerate_mock.call_args.kwargs["layout_static_only"])+ self.assertEqual(summary["profile"], "layout-testharness")+ self.assertEqual(summary["viewport"]["width"], 800)+ def test_main_uses_wpt_harness_multipliers_independent_of_outer_deadline(self) -> None:+ calls = []++ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.base_url = "http://127.0.0.1:8000"+ self.alternate_base_url = "http://127.0.0.1:8001"+ self.external_base_url = None+ self.external_alternate_base_url = None+ self.external_remote_base_url = None+ self.external_host = None++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ calls.append((multipliers, default_multiplier))++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ return f"{self.base_url}/{case_path}"++ result_dict = {+ "engine": "moli",+ "binary": "/tmp/moli",+ "binary_sha256": "sha",+ "binary_version": "0.1.0",+ "endpoint": "cli:/tmp/moli",+ "ready_ms": None,+ "setup_error": None,+ "cases": [],+ }+ cases = [+ WptCase("normal.html", timeout_multiplier=1.0),+ WptCase("long.html", timeout_multiplier=LONG_TIMEOUT_MULTIPLIER),+ WptCase(+ "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html",+ timeout_multiplier=1.0,+ ),+ ]++ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir) / "out"+ with (+ patch(+ "moli_benchmark.wpt_cross.__main__.enumerate_cases",+ return_value=cases,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.build_driver",+ return_value=SimpleNamespace(cli_fetch_command=["moli"]),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",+ return_value=SimpleNamespace(setup_error=None),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",+ return_value=result_dict,+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",+ Path(temp_dir) / "repo-case-lists",+ ),+ patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),+ redirect_stdout(StringIO()),+ redirect_stderr(StringIO()),+ ):+ code = main(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ str(output_dir),+ ]+ )++ self.assertEqual(code, 0)+ self.assertEqual(+ calls,+ [+ (+ {+ "normal.html": 1.0,+ "long.html": LONG_TIMEOUT_MULTIPLIER,+ "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html": 1.0,+ },+ 1.0,+ )+ ],+ )+ def test_normal_dynamic_import_uses_normal_wpt_timeout(self) -> None:+ case = WptCase(+ "html/semantics/scripting-1/the-script-element/module/dynamic-import/delay-load-event.html"+ )++ self.assertEqual(_harness_timeout_multiplier(case), 1.0)+ def test_parser_rejects_removed_timeout_flags_and_legacy_parallelism(self) -> None:+ parser = _build_parser()+ base = [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "chrome",+ "--output-dir",+ "/tmp/out",+ ]+ for removed_flag in (+ "--case-timeout",+ "--case-timeout-engine",+ "--cdp-parallelism",+ ):+ with self.subTest(removed_flag=removed_flag), self.assertRaises(SystemExit):+ parser.parse_args([*base, removed_flag, "1"])++ for invalid_parallelism in ("0", "-1", "not-an-integer"):+ with self.subTest(invalid_parallelism=invalid_parallelism), self.assertRaises(+ SystemExit+ ):+ parser.parse_args(+ [*base, "--parallelism", invalid_parallelism]+ )+ def test_main_matrix_preserves_harness_message(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ manifest = root / "known.json"+ output_dir = root / "out"+ manifest.write_text(+ json.dumps(+ {+ "engine": "moli",+ "rules": [+ {+ "case_path": "known.html",+ "expected_status": "fail",+ "message_contains": "tracked message",+ }+ ],+ }+ ),+ encoding="utf-8",+ )++ code = self._run_wpt_cross_with_fake_moli(+ output_dir=output_dir,+ known_failures=manifest,+ case_status="fail",+ failure_message="expected 555 but got 100",+ )++ self.assertEqual(code, 0)+ matrix = json.loads((output_dir / "matrix.json").read_text(encoding="utf-8"))+ result = matrix[0]["results"]["moli"]+ self.assertEqual(+ result["harness_message"],+ "Harness completed with a tracked message",+ )+ audit = json.loads(+ (output_dir / "known-failure-audit-moli.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertEqual(+ audit["known_failures"][0]["harness_message"],+ "Harness completed with a tracked message",+ )+ def test_build_partial_preserves_diagnostic_fields(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ output_dir = Path(temp_dir)+ (output_dir / "engine-moli.json").write_text(+ json.dumps(+ {+ "cases": [+ {+ "case_path": "known.html",+ "status": "fail",+ "duration_ms": 1.0,+ "subtests": {+ "total": 0,+ "pass": 0,+ "fail": 0,+ "timeout": 0,+ "notrun": 0,+ },+ "harness_status_name": "ERROR",+ "harness_message": "Unhandled rejection: cycle",+ "error": "testharness completed without reporting any subtests",+ "test_type": "reftest",+ "failures": [{"name": "== known-ref.html"}],+ "failure_names": ["== known-ref.html"],+ "reftest_comparisons": [+ {+ "reference_path": "known-ref.html",+ "relation": "==",+ "passed": False,+ "max_difference": 255,+ "different_pixels": 10,+ }+ ],+ "artifacts": {+ "test": "artifacts/moli/known/test.png",+ "references": [+ {+ "reference": "artifacts/moli/known/reference-01.png",+ "diff": "artifacts/moli/known/diff-01.png",+ }+ ],+ },+ }+ ]+ }+ ),+ encoding="utf-8",+ )++ with redirect_stdout(StringIO()):+ code = build_partial_main([str(output_dir), "--engine", "moli"])++ self.assertEqual(code, 0)+ matrix = json.loads(+ (output_dir / "matrix.partial.moli.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertEqual(+ matrix[0]["results"]["moli"]["harness_message"],+ "Unhandled rejection: cycle",+ )+ self.assertEqual(matrix[0]["test_type"], "reftest")+ result = matrix[0]["results"]["moli"]+ self.assertEqual(result["failure_names"], ["== known-ref.html"])+ self.assertEqual(result["reftest_comparisons"][0]["relation"], "==")+ self.assertEqual(+ result["artifacts"]["test"],+ "artifacts/moli/known/test.png",+ )+ def test_parser_accepts_known_failure_audit_options(self) -> None:+ parser = _build_parser()+ args = parser.parse_args(+ [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ "/tmp/out",+ "--known-failures",+ "/tmp/known.json",+ "--known-failures-engine",+ "moli",+ "--allow-missing-known-failures",+ ]+ )++ self.assertEqual(args.known_failures, Path("/tmp/known.json"))+ self.assertEqual(args.known_failures_engine, "moli")+ self.assertTrue(args.allow_missing_known_failures)+ def test_main_writes_known_failure_audit_when_manifest_matches(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ docs = root / "docs"+ wpt = root / "wpt"+ docs.mkdir()+ wpt.mkdir()+ (docs / "wasm-current.md").write_text("# wasm\n", encoding="utf-8")+ (wpt / "known.html").write_text("<!doctype html>", encoding="utf-8")+ manifest = root / "known.json"+ output_dir = root / "out"+ manifest.write_text(+ json.dumps(+ {+ "engine": "moli",+ "categories": {+ "wasm": {+ "tracking_doc": "docs/wasm-current.md",+ "scope": "tracked wasm failure",+ "evidence": [+ {+ "kind": "doc",+ "path": "docs/wasm-current.md",+ "note": "local fixture evidence",+ },+ {+ "kind": "wpt",+ "path": "wpt/known.html",+ "note": "known failure source fixture",+ }+ ],+ }+ },+ "rules": [+ {+ "case_path": "known.html",+ "category": "wasm",+ "expected_status": "fail",+ "message_contains": "expected 555",+ "reason": "tracked wasm failure",+ }+ ],+ }+ ),+ encoding="utf-8",+ )++ code = self._run_wpt_cross_with_fake_moli(+ output_dir=output_dir,+ known_failures=manifest,+ case_status="fail",+ )++ self.assertEqual(code, 0)+ audit = json.loads(+ (output_dir / "known-failure-audit-moli.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertTrue(audit["ok"])+ self.assertEqual(audit["counts"]["known_failures"], 1)+ self.assertEqual(audit["categories"]["wasm"]["scope"], "tracked wasm failure")+ summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))+ self.assertTrue(summary["known_failure_audits"]["moli"]["ok"])+ self.assertEqual(+ summary["known_failure_audits"]["moli"]["categories"]["wasm"][+ "tracking_doc"+ ],+ "docs/wasm-current.md",+ )+ self.assertEqual(+ summary["known_failure_audits"]["moli"]["category_counts"][+ "known_failures"+ ],+ {"wasm": 1},+ )+ def test_main_can_skip_missing_known_failures_for_focused_runs(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ manifest = root / "known.json"+ output_dir = root / "out"+ manifest.write_text(+ json.dumps(+ {+ "engine": "moli",+ "rules": [+ {+ "case_path": "known.html",+ "expected_status": "fail",+ "message_contains": "expected 555",+ },+ {+ "case_path": "not-run.html",+ "expected_status": "fail",+ },+ ],+ }+ ),+ encoding="utf-8",+ )++ code = self._run_wpt_cross_with_fake_moli(+ output_dir=output_dir,+ known_failures=manifest,+ case_status="fail",+ allow_missing_known_failures=True,+ )++ self.assertEqual(code, 0)+ audit = json.loads(+ (output_dir / "known-failure-audit-moli.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertTrue(audit["ok"])+ self.assertEqual(audit["counts"]["known_failures"], 1)+ self.assertEqual(audit["counts"]["missing_expected_failures"], 0)+ self.assertEqual(audit["counts"]["skipped_known_failures"], 1)+ summary = json.loads((output_dir / "summary.json").read_text(encoding="utf-8"))+ self.assertEqual(+ summary["known_failure_audits"]["moli"]["counts"][+ "skipped_known_failures"+ ],+ 1,+ )+ def test_main_returns_nonzero_when_known_failure_audit_finds_unexpected_failure(+ self,+ ) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ manifest = root / "known.json"+ output_dir = root / "out"+ manifest.write_text(+ json.dumps({"engine": "moli", "rules": []}), encoding="utf-8"+ )++ code = self._run_wpt_cross_with_fake_moli(+ output_dir=output_dir,+ known_failures=manifest,+ case_status="fail",+ )++ self.assertEqual(code, 5)+ audit = json.loads(+ (output_dir / "known-failure-audit-moli.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertFalse(audit["ok"])+ self.assertEqual(audit["counts"]["unexpected_failures"], 1)+ def test_main_returns_nonzero_when_known_failure_is_resolved(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ manifest = root / "known.json"+ output_dir = root / "out"+ manifest.write_text(+ json.dumps(+ {+ "engine": "moli",+ "rules": [+ {+ "case_path": "known.html",+ "expected_status": "fail",+ }+ ],+ }+ ),+ encoding="utf-8",+ )++ code = self._run_wpt_cross_with_fake_moli(+ output_dir=output_dir,+ known_failures=manifest,+ case_status="pass",+ )++ self.assertEqual(code, 5)+ audit = json.loads(+ (output_dir / "known-failure-audit-moli.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertFalse(audit["ok"])+ self.assertEqual(audit["counts"]["resolved_known_failures"], 1)diff --git a/moli-benchmark/tests/wpt_cross_test_support.py b/moli-benchmark/tests/wpt_cross_test_support.pynew file mode 100644index 000000000..a4cccf180--- /dev/null+++ b/moli-benchmark/tests/wpt_cross_test_support.py@@ -0,0 +1,272 @@+from __future__ import annotations++import asyncio+import hashlib+import io+import json+import socket+import subprocess+import tempfile+import threading+import time+import unittest+from contextlib import redirect_stderr, redirect_stdout+from http.client import HTTPConnection+from io import StringIO+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock, patch+from urllib.request import Request, urlopen++from PIL import Image++from moli_benchmark.config import clear_current_proxy_env+from moli_benchmark.raw_cdp import RawCdpError+from moli_benchmark.wpt_cross.__main__ import (+ CASE_LIST_FILES,+ WPT_CROSS_CASE_TIMEOUT_SECONDS,+ WPT_CROSS_PARALLELISM,+ _build_parser,+ _case_requires_non_trustworthy_origin,+ _case_requires_trustworthy_origin,+ _deduplicate_cases,+ _harness_timeout_multiplier,+ _is_full_case_list_run,+ _recorded_failure_drift,+ _url_for_case_origin,+ _write_repo_case_lists,+ main,+)+from moli_benchmark.wpt_cross.build_partial import main as build_partial_main+from moli_benchmark.wpt_cross.case_set import (+ ANY_JS_WINDOW_QUERY,+ WINDOW_JS_WINDOW_QUERY,+ any_js_window_case_path,+ DEFAULT_EXCLUDE_DIR_PREFIXES,+ FuzzyTolerance,+ LAYOUT_PROFILE_DIR_PREFIXES,+ LONG_TIMEOUT_MULTIPLIER,+ ReftestReference,+ WptCase,+ enumerate_cases,+ enumerate_reftest_cases,+ explicit_reftest_case,+ explicit_case,+ parse_any_js_meta,+ window_js_window_case_path,+)+from moli_benchmark.wpt_cross.engine import (+ _moli_command,+ _moli_fetch,+ _lightpanda_fetch,+)+from moli_benchmark.wpt_cross.cli_runner import (+ MOLI_WPT_USER_AGENT,+ _CliCaseWorkerInput,+ _CliSubprocessResult,+ _classify_cli_case_result,+ _moli_fixture_host_resolve_args,+ _nonzero_exit_status,+ _payload_from_stdout_html,+ _payload_grace_for_process_result,+ _run_cli_case_worker,+ _stderr_tail,+ run_engine_on_cases_cli,+)+from moli_benchmark.wpt_cross.any_js import (+ ANY_JS_DEDICATED_WORKER_GLOBAL,+ ANY_JS_WINDOW_GLOBAL,+ any_js_case_path_for_global,+ any_js_source_script_path,+ any_js_worker_script_path,+)+from moli_benchmark.wpt_cross.render_html import render_html+from moli_benchmark.wpt_cross.runner import (+ _AttachedPage,+ _PageSessionUnusable,+ _ReftestEvidence,+ _close_page,+ _has_navigation_evidence,+ _navigation_identity,+ _normalized_case_path,+ _normalized_navigation_url,+ _run_async,+ _run_one_case,+ _write_reftest_failure_artifacts,+ CapturedScreenshot,+ CaseResult,+ EngineRunResult,+ LAYOUT_VIEWPORT,+ ReftestReferenceRun,+ ReftestRun,+ case_result_to_dict,+ classify_payload,+ compare_reftest_screenshots,+ reftest_comparisons_pass,+ reftest_relation_passes,+)+from moli_benchmark.wpt_cross.scheduler import (+ FIXED_RUN_SHUFFLE_SEED,+ build_run_schedule,+)+from moli_benchmark.wpt_cross.server import (+ _fetch_status_response,+ BENCH_REPORT_BRIDGE,+ BENCH_TESTDRIVER_VENDOR_BRIDGE,+ BENCH_TIMEOUT_MULTIPLIER_QUERY,+ WptFixtureServer,+ ResultsStore,+ _apply_header_operations,+ _any_js_window_wrapper,+ _bench_report_bridge,+ _content_security_policy_resource_response,+ _workers_modules_export_on_load_script_response,+ _inject_bench_report_bridge_config,+ _host_header_hostname,+ _headers_include,+ _normalize_harness_case_key,+ _needs_wpt_template_substitution,+ _legacy_wpt_resource_alias,+ _pipe_response_header_operations,+ _pipe_response_status,+ _pipe_trickle_delay_seconds,+ _redirect_fixture_response,+ _response_content_type_and_extra_headers,+ _resolve_wpt_static_script_url,+ _sidecar_response_headers,+ _static_response_header_block,+ _static_response_headers,+ _substitute_wpt_template_variables,+ _window_js_window_wrapper,+ _wasm_webapi_status_code,+ _wpt_delay_seconds,+ _wpt_dedicated_worker_js_wrapper_html,+ _wpt_any_dedicated_worker_wrapper_html,+ _wpt_any_dedicated_worker_wrapper_js,+ _wpt_any_window_wrapper_html,+ _wpt_window_js_wrapper_html,+)+++# WPT fixture tests use loopback servers; inherited shell proxies can intercept+# urllib requests to those servers, so normalize the script process once.+clear_current_proxy_env()+++++class WptCrossTestCase(unittest.TestCase):+ def _run_wpt_cross_with_fake_moli(+ self,+ *,+ output_dir: Path,+ known_failures: Path,+ case_status: str,+ failure_message: str = "expected 555 but got 100",+ allow_missing_known_failures: bool = False,+ extra_args: list[str] | None = None,+ ) -> int:+ class FakeServer:+ def __init__(self, wpt_root: Path) -> None:+ self.wpt_root = Path(wpt_root)+ self.base_url = "http://127.0.0.1:8000"+ self.alternate_base_url = "http://127.0.0.1:8001"+ self.external_base_url = None+ self.external_alternate_base_url = None+ self.external_remote_base_url = None+ self.external_host = None++ def __enter__(self) -> "FakeServer":+ return self++ def __exit__(self, *args: object) -> None:+ return None++ def set_harness_timeout_multipliers(+ self,+ multipliers: dict[str, float],+ *,+ default_multiplier: float,+ ) -> None:+ return None++ def url_for_case(self, case_path: str, *, external: bool = False) -> str:+ return f"{self.base_url}/{case_path}"++ failures = []+ if case_status != "pass":+ failures = [{"name": "subtest", "message": failure_message}]+ result_dict = {+ "engine": "moli",+ "binary": "/tmp/moli",+ "binary_sha256": "sha",+ "binary_version": "0.1.0",+ "endpoint": "cli:/tmp/moli",+ "ready_ms": None,+ "setup_error": None,+ "cases": [+ {+ "case_path": "known.html",+ "status": case_status,+ "duration_ms": 1.0,+ "subtests": {+ "total": 1,+ "pass": int(case_status == "pass"),+ "fail": int(case_status != "pass"),+ "timeout": 0,+ "notrun": 0,+ },+ "failures": failures,+ "harness_status_name": "OK",+ "harness_message": (+ "Harness completed with a tracked message"+ if case_status != "pass"+ else None+ ),+ "error": None,+ }+ ],+ }+ with (+ patch(+ "moli_benchmark.wpt_cross.__main__.enumerate_cases",+ return_value=[WptCase("known.html")],+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.build_driver",+ return_value=SimpleNamespace(cli_fetch_command=["moli"]),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.run_engine_on_cases_cli",+ return_value=SimpleNamespace(setup_error=None),+ ),+ patch(+ "moli_benchmark.wpt_cross.__main__.engine_result_to_dict",+ return_value=result_dict,+ ),+ patch("moli_benchmark.wpt_cross.server.WptFixtureServer", FakeServer),+ patch(+ "moli_benchmark.wpt_cross.__main__.REPO_CASE_LIST_DIR",+ output_dir.parent / "repo-case-lists",+ ),+ redirect_stdout(StringIO()),+ redirect_stderr(StringIO()),+ ):+ args = [+ "--wpt-root",+ "/tmp/wpt",+ "--engine",+ "moli",+ "--output-dir",+ str(output_dir),+ "--known-failures",+ str(known_failures),+ ]+ if allow_missing_known_failures:+ args.append("--allow-missing-known-failures")+ if extra_args is not None:+ args.extend(extra_args)+ return main(args)+++__all__ = [name for name in globals() if not name.startswith("__")]