Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): serve XHR request URI and redirect fixtures
ldm07ec4a3ac15fe
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex 36ceaa993..ac30d06a9 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -833,6 +833,24 @@ def _remote_context_handler_reference_patterns(directory: str) -> tuple[re.Patte)++@lru_cache(maxsize=None)+def _xhr_url_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:+ references = []+ for name in ("requri.py", "redirect.py"):+ resource = f"xhr/resources/{name}"+ relative = posixpath.relpath(resource, directory)+ references.extend(("/" + resource, relative, "./" + relative))+ return tuple(+ re.compile(+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"+ )+ for reference in references+ )+++def _supported_wptserve_handler_references(rel: str | None,) -> tuple[re.Pattern[str], ...]:@@ -862,6 +880,7 @@ def _supported_wptserve_handler_references(supported += _script_load_error_handler_reference_patterns(posixpath.dirname(rel) or ".")if rel is not None and rel.startswith("xhr/"):supported += SUPPORTED_XHR_DELAY_WPTSERVE_HANDLER_PATTERNS+ supported += _xhr_url_handler_reference_patterns(rel.rsplit("/", 1)[0])if rel is not None and rel.startswith("html/semantics/scripting-1/the-script-element/module/"):diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 2c1fe7c06..f6b20cee5 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -130,6 +130,10 @@ IFRAME_STASH_PATH = "/html/semantics/embedded-content/the-iframe-element/stash.pCOMMON_ECHO_PATH = "/common/echo.py"COMMON_REDIRECT_PATH = "/common/redirect.py"FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py"+XHR_URL_RESOURCE_PATHS = {+ "/xhr/resources/requri.py",+ "/xhr/resources/redirect.py",+}FETCH_ABORT_RESOURCE_PATHS = {"/fetch/api/resources/stash-put.py","/fetch/api/resources/stash-take.py",@@ -1261,6 +1265,32 @@ def _form_submission_response(return b"OK" if valid else b"FAIL"+def _xhr_redirect_fixture_response(+ path: str, query: str+) -> tuple[int, str | None, list[tuple[str, str]], bytes, float | None]:+ """Model xhr/resources/redirect.py, including its second Location decode."""+ params = parse_qs(query, keep_blank_values=True, encoding="latin-1")+ code = int(params.get("code", ["302"])[0])+ location = params.get("location", [path + "?followed"])[0]+ location = location.encode("latin-1").decode("utf-8")+ if location:+ location = parse_qs("location=" + location)["location"][0]+ if location.startswith("redirect.py"):+ location += "&code=" + str(code)+ delay = None+ if "delay" in params:+ delay = float(params["delay"][0]) / 1_000+ if not math.isfinite(delay) or delay < 0:+ raise ValueError("invalid redirect delay")+ if "followed" in params:+ # Preserve the upstream handler's header spelling.+ return 200, None, [("Content:Type", "text/plain")], b"MAGIC HAPPENED", delay+ if any(character in location for character in "\r\n"):+ raise ValueError("invalid Location header")+ location.encode("latin-1")+ return code, "WEBSRT MARKETING", [("Location", location)], b"TEST", delay++def _redirect_fixture_response(query: str) -> tuple[int, str] | None:"""Return the shared redirect response used by static WPT fixture handlers."""@@ -2048,6 +2078,9 @@ requestExecutor("{executor_uuid}", {start_on_js});return True+ def _serve_xhr_url_method(self) -> None:+ self._serve_xhr_url_resource(emit_body=self.command != "HEAD")+def do_GET(self) -> None: # noqa: N802 (BaseHTTPRequestHandler API)if self.headers.get("Upgrade", "").lower() == "websocket":self._serve_websocket()@@ -2081,7 +2114,10 @@ requestExecutor("{executor_uuid}", {start_on_js});unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",)return- if self._serve_xhr_response_resource():+ if (+ self._serve_xhr_response_resource()+ or self._serve_xhr_url_resource(emit_body=True)+ ):returnparsed = urlparse(self.path)path = unquote(parsed.path)@@ -2134,7 +2170,10 @@ requestExecutor("{executor_uuid}", {start_on_js});unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",)return- if self._serve_xhr_response_resource():+ if (+ self._serve_xhr_response_resource()+ or self._serve_xhr_url_resource(emit_body=True)+ ):returnparsed = urlparse(self.path)path = unquote(parsed.path)@@ -2219,7 +2258,10 @@ requestExecutor("{executor_uuid}", {start_on_js});unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",)return- if self._serve_xhr_response_resource():+ if (+ self._serve_xhr_response_resource()+ or self._serve_xhr_url_resource(emit_body=True)+ ):returnparsed = urlparse(self.path)path = unquote(parsed.path)@@ -2278,6 +2320,8 @@ requestExecutor("{executor_uuid}", {start_on_js});unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",)return+ if self._serve_xhr_url_resource(emit_body=True):+ returnparsed = urlparse(self.path)if unquote(parsed.path) == NAVIGATION_SECOND_VISIT_PATH:self._serve_navigation_second_visit()@@ -2477,7 +2521,10 @@ requestExecutor("{executor_uuid}", {start_on_js});unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",)return- if self._serve_xhr_response_resource(emit_body=emit_body):+ if (+ self._serve_xhr_response_resource(emit_body=emit_body)+ or self._serve_xhr_url_resource(emit_body=emit_body)+ ):returnparsed = urlparse(self.path)path = unquote(parsed.path)@@ -3216,6 +3263,52 @@ requestExecutor("{executor_uuid}", {start_on_js});self.send_error(500)+ def _serve_xhr_url_resource(self, *, emit_body: bool) -> bool:+ parsed = urlsplit(self.path)+ path = unquote(parsed.path)+ if path not in XHR_URL_RESOURCE_PATHS:+ return False+ try:+ if path == "/xhr/resources/requri.py":+ params = parse_qs(parsed.query, keep_blank_values=True)+ uri = self.path+ if "full" in params and not uri.startswith("http://"):+ authority = self.headers.get("Host")+ if authority is None:+ authority = _url_host_literal(+ str(self.server.server_address[0])+ )+ if urlsplit("//" + authority).port is None:+ authority += ":" + str(self.server.server_address[1])+ uri = f"http://{authority}{uri}"+ status, reason, headers, body = 200, None, [], uri.encode("utf-8")+ else:+ status, reason, headers, body, delay = _xhr_redirect_fixture_response(+ parsed.path, parsed.query+ )+ if delay is not None:+ time.sleep(delay)+ except (ValueError, KeyError, OverflowError):+ self.send_error(500)+ return True+ # These handlers reply without reading uploads. Close connections+ # with unread bodies so a redirect can arrive before upload finishes.+ if (+ self.headers.get("Transfer-Encoding") is not None+ or self.headers.get("Content-Length", "0").strip() not in {"", "0"}+ ):+ self.close_connection = True+ headers.append(("Connection", "close"))+ self._send_bytes(+ None,+ body,+ emit_body=emit_body,+ extra_headers=headers,+ status_code=status,+ status_text=reason,+ )+ return True+def _serve_xhr_delay(self, query: str, *, emit_body: bool) -> None:delay_seconds = _wpt_delay_seconds(query)if delay_seconds is None:@@ -3269,6 +3362,9 @@ requestExecutor("{executor_uuid}", {start_on_js});return self._serve_document_charset_resourceif name.startswith("do_") and unquote(urlsplit(self.path).path) == IFRAME_STASH_PATH:return self._serve_iframe_stash_resource++ if name.startswith("do_") and unquote(urlsplit(self.path).path) in XHR_URL_RESOURCE_PATHS:+ return self._serve_xhr_url_methodif name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_ECHO_PATH:return self._serve_common_echo_resourceif name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_REDIRECT_PATH:@@ -3908,12 +4004,15 @@ requestExecutor("{executor_uuid}", {start_on_js});cache_control: str | None = "no-store",auto_content_length: bool = True,) -> None:- content_type, extra_headers = _response_content_type_and_extra_headers(- content_type,- extra_headers,- )+ if content_type is None:+ header_block = list(extra_headers or [])+ else:+ content_type, extra_headers = _response_content_type_and_extra_headers(+ content_type,+ extra_headers,+ )+ header_block = _static_response_header_block(content_type, extra_headers)self.send_response(status_code, status_text)- header_block = _static_response_header_block(content_type, extra_headers)for name, value in header_block:self.send_header(name, value)if auto_content_length and not _headers_include(header_block, "Content-Length"):diff --git a/moli-benchmark/tests/test_wpt_xhr_url_server.py b/moli-benchmark/tests/test_wpt_xhr_url_server.pynew file mode 100644index 000000000..ad4bc2428--- /dev/null+++ b/moli-benchmark/tests/test_wpt_xhr_url_server.py@@ -0,0 +1,218 @@+from __future__ import annotations++import tempfile+import unittest+from contextlib import ExitStack+from http.client import HTTPConnection+from pathlib import Path+from unittest.mock import patch+from urllib.parse import urlencode++from moli_benchmark.wpt_cross.case_set import enumerate_cases+from moli_benchmark.wpt_cross.server import WptFixtureServer+++RESOURCE_DIR = "/xhr/resources/"+++class XhrUrlFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ (self.root / "resources/testharnessreport.js").write_text("// report")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None+ ))++ def server(self, *, keep_alive: bool = False) -> WptFixtureServer:+ server = WptFixtureServer(self.root)+ if keep_alive:+ server.httpd.RequestHandlerClass.protocol_version = "HTTP/1.1"+ return self.stack.enter_context(server)++ def request(+ self, port: int, target: str, *, method: str = "GET",+ headers: dict[str, str] | None = None,+ ) -> tuple[int, str, dict[str, str], bytes]:+ connection = HTTPConnection("127.0.0.1", port, timeout=2)+ try:+ connection.request(method, target, headers=headers or {})+ response = connection.getresponse()+ return (+ response.status, response.reason,+ {name.lower(): value for name, value in response.getheaders()},+ response.read(),+ )+ finally:+ connection.close()++ def test_request_uri_preserves_raw_path_and_query(self) -> None:+ server = self.server()+ for path in (+ "requri.py", "requri.py?", "requri.py?help=",+ "requri.py?%23foobar", "%72equri.py?a=%2f%23&b=+%20",+ ):+ with self.subTest(path=path):+ target = RESOURCE_DIR + path+ status, _, headers, body = self.request(server.port, target)+ self.assertEqual((status, body), (200, target.encode()))+ self.assertNotIn("content-type", headers)++ for query in ("full", "full=", "full=0", "full&full=ignored"):+ with self.subTest(query=query):+ target = RESOURCE_DIR + "requri.py?" + query+ for host, authority in (+ ("example.test:1234", "example.test:1234"),+ ("example.test", f"example.test:{server.port}"),+ ):+ status, _, _, body = self.request(+ server.port, target, headers={"Host": host}+ )+ self.assertEqual(+ (status, body), (200, f"http://{authority}{target}".encode())+ )++ target = "http://example.test:1234" + RESOURCE_DIR + "requri.py?full"+ status, _, _, body = self.request(server.port, target)+ self.assertEqual((status, body), (200, target.encode()))++ def test_fixtures_accept_custom_methods_and_head_omits_only_body(self) -> None:+ server = self.server()+ for method in ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "YO", "CHICKEN"):+ for resource, status, expected in (+ ("requri.py?x=%23", 200, b"/xhr/resources/requri.py?x=%23"),+ ("redirect.py", 302, b"TEST"),+ ):+ with self.subTest(method=method, resource=resource):+ actual, _, headers, body = self.request(+ server.port, RESOURCE_DIR + resource, method=method+ )+ self.assertEqual(actual, status)+ self.assertEqual(body, b"" if method == "HEAD" else expected)+ self.assertEqual(headers["content-length"], str(len(expected)))+ self.assertNotIn("content-type", headers)+ status, _, _, _ = self.request(server.port, "/unhandled.py", method="CHICKEN")+ self.assertEqual(status, 501)++ def test_redirect_status_location_and_first_query_values(self) -> None:+ server = self.server()+ cases = [+ ("", 302, RESOURCE_DIR + "redirect.py?followed"),+ ("code=307&code=301&location=first&location=second", 307, "first"),+ ("location=%252Ftarget%253Fx%253D1%2523fragment", 302, "/target?x=1#fragment"),+ ("location=redirect.py%3Flocation%3Dcontent.py&code=308", 308,+ "redirect.py?location=content.py&code=308"),+ ("location=", 302, ""),+ (urlencode({"location": "javascript:invalid"}), 302, "javascript:invalid"),+ (urlencode({"location": "http://example.test/target?q=1"}), 302,+ "http://example.test/target?q=1"),+ ]+ cases.extend((f"code={code}", code, RESOURCE_DIR + "redirect.py?followed")+ for code in (200, 300, 301, 303, 399))+ for query, code, location in cases:+ with self.subTest(query=query):+ status, reason, headers, body = self.request(+ server.port, RESOURCE_DIR + "redirect.py?" + query+ )+ self.assertEqual((status, reason, body), (code, "WEBSRT MARKETING", b"TEST"))+ self.assertEqual(headers["location"], location)+ self.assertNotIn("content-type", headers)++ def test_redirect_followed_response_and_delay_match_upstream(self) -> None:+ server = self.server()+ _, _, redirect_headers, _ = self.request(server.port, RESOURCE_DIR + "redirect.py")+ status, _, headers, body = self.request(server.port, redirect_headers["location"])+ self.assertEqual((status, body), (200, b"MAGIC HAPPENED"))+ # Upstream spells this header "Content:Type"; it is not Content-Type.+ self.assertEqual(headers["content"], "Type: text/plain")+ self.assertNotIn("content-type", headers)+ self.assertNotIn("location", headers)+ # Once followed, the parsed location is no longer used as a header.+ status, _, headers, body = self.request(+ server.port, RESOURCE_DIR + "redirect.py?followed&location=%E2%82%AC"+ )+ self.assertEqual((status, body), (200, b"MAGIC HAPPENED"))+ self.assertNotIn("location", headers)+ with patch("moli_benchmark.wpt_cross.server.time.sleep") as sleep:+ for query, method in (+ ("delay=250&delay=1000", "GET"),+ ("delay=250&followed=", "GET"),+ ("delay=250", "HEAD"),+ ):+ self.request(server.port, RESOURCE_DIR + "redirect.py?" + query, method=method)+ sleep.assert_called_once_with(0.25)+ sleep.reset_mock()+ for query in (+ "code=invalid", "code=", "delay=invalid", "delay=nan", "delay=inf",+ "delay=-1", "location=%FF", "location=%250D%250A",+ "code=invalid&followed",+ ):+ with self.subTest(query=query):+ status, _, _, _ = self.request(server.port, RESOURCE_DIR + "redirect.py?" + query)+ self.assertEqual(status, 500)++ def test_response_can_arrive_before_upload_body(self) -> None:+ server = self.server(keep_alive=True)+ for resource, expected in (+ ("redirect.py", (302, b"TEST")),+ ("requri.py", (200, b"/xhr/resources/requri.py")),+ ):+ for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")):+ with self.subTest(resource=resource, framing=framing):+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.putrequest("POST", RESOURCE_DIR + resource)+ connection.putheader(*framing)+ connection.endheaders()+ # No upload bytes have been sent when we read the response.+ response = connection.getresponse()+ self.assertEqual((response.status, response.read()), expected)+ self.assertEqual(response.headers["Connection"], "close")+ self.assertTrue(response.will_close)+ finally:+ connection.close()++ def test_bodyless_requests_can_reuse_connection(self) -> None:+ server = self.server(keep_alive=True)+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.request("GET", RESOURCE_DIR + "redirect.py")+ redirect = connection.getresponse()+ self.assertEqual((redirect.status, redirect.read()), (302, b"TEST"))+ first_socket = connection.sock+ self.assertIsNotNone(first_socket)+ connection.request("CHICKEN", redirect.headers["Location"])+ followed = connection.getresponse()+ self.assertEqual((followed.status, followed.read()), (200, b"MAGIC HAPPENED"))+ self.assertIs(connection.sock, first_socket)+ finally:+ connection.close()++ def test_case_selection_resolves_only_supported_xhr_resource_paths(self) -> None:+ sources = {+ "xhr/absolute.html": "fetch('/xhr/resources/requri.py?full');",+ "xhr/relative.html": "fetch('resources/requri.py'); fetch('./resources/redirect.py');",+ "xhr/nested/parent.html": "fetch('../resources/redirect.py');",+ "xhr/unsupported.html": "fetch('resources/requri.py'); fetch('resources/other.py');",+ "xhr/suffix.html": "fetch('resources/redirect.py-extra');",+ "xhr/script-suffix.html": "fetch('resources/redirect.py.js');",+ "xhr/prefix.html": "fetch('/wrong/xhr/resources/redirect.py');",+ "xhr/wrong-parent.html": "fetch('../resources/redirect.py');",+ "xhr/wrong-root.html": "fetch('redirect.py');",+ "xhr/nested/wrong-relative.html": "fetch('resources/redirect.py');",+ }+ for path, script in sources.items():+ target = self.root / path+ target.parent.mkdir(parents=True, exist_ok=True)+ target.write_text(+ '<script src="/resources/testharness.js"></script>'+ '<script src="/resources/testharnessreport.js"></script>'+ f"<script>{script}</script>"+ )+ selected = enumerate_cases(self.root, dir_prefixes=("xhr",))+ self.assertEqual([case.case_path for case in selected], [+ "xhr/absolute.html", "xhr/nested/parent.html", "xhr/relative.html",+ ])