Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): serve XHR response status and last-modified fixtures
ldm065616592dc00
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex 319393400..894434f5e 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -746,9 +746,27 @@ def _supported_wptserve_handler_references("html/semantics/scripting-1/the-script-element/module/"):supported += SUPPORTED_MODULE_DELAY_WPTSERVE_HANDLER_PATTERNS+ if rel is not None:+ supported += _xhr_response_handler_reference_patterns(posixpath.dirname(rel) or ".")return supported+@lru_cache(maxsize=None)+def _xhr_response_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:+ references = []+ for name in ("status.py", "last-modified.py"):+ resource = f"xhr/resources/{name}"+ relative = posixpath.relpath(resource, directory)+ references.extend(("/" + resource, relative, "./" + relative))+ return tuple(+ re.compile(+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"+ )+ for reference in references+ )++def _has_unsupported_server_feature(text: str,*,diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 19e2f4d4c..4ffa6a0db 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -37,6 +37,7 @@ import threadingimport timeimport uuidfrom html import escape as html_escape+from email.utils import formatdatefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Pathfrom urllib.parse import parse_qs, parse_qsl, unquote, urlparse, urlsplit, urlunsplit@@ -71,6 +72,10 @@ http.client._MAXHEADERS = 512DEFAULT_TESTHARNESS_TIMEOUT_SECONDS = 10.0MAX_REQUEST_BODY_BYTES = 16 * 1024 * 1024MAX_REQUEST_BODY_LINE_BYTES = 64 * 1024+XHR_RESPONSE_RESOURCE_PATHS = {+ "/xhr/resources/status.py",+ "/xhr/resources/last-modified.py",+}FETCH_ABORT_RESOURCE_PATHS = {"/fetch/api/resources/stash-put.py","/fetch/api/resources/stash-take.py",@@ -1602,6 +1607,8 @@ def _make_handler(self._serve(emit_body=False)def do_OPTIONS(self) -> None: # noqa: N802+ if self._serve_xhr_response_resource():+ returnparsed = urlparse(self.path)path = unquote(parsed.path)if path in FETCH_ABORT_RESOURCE_PATHS | {@@ -1627,6 +1634,8 @@ def _make_handler(self.send_error(404)def do_POST(self) -> None: # noqa: N802+ if self._serve_xhr_response_resource():+ returnparsed = urlparse(self.path)path = unquote(parsed.path)if path in FETCH_ABORT_RESOURCE_PATHS | {@@ -1673,6 +1682,8 @@ def _make_handler(self.end_headers()def _serve_fetch_resource_method(self) -> None:+ if self._serve_xhr_response_resource():+ returnparsed = urlparse(self.path)if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS:self._serve_fetch_abort_resource(@@ -1819,6 +1830,8 @@ def _make_handler(returndef _serve(self, *, emit_body: bool) -> None:+ if self._serve_xhr_response_resource(emit_body=emit_body):+ returnparsed = urlparse(self.path)path = unquote(parsed.path)if path == (@@ -2346,6 +2359,39 @@ def _make_handler(emit_body=emit_body,)+ def __getattr__(self, name: str):+ if name.startswith("do_") and unquote(urlparse(self.path).path) in XHR_RESPONSE_RESOURCE_PATHS:+ return self._serve_xhr_response_resource+ raise AttributeError(name)++ def _serve_xhr_response_resource(self, *, emit_body: bool = True) -> bool:+ parsed = urlparse(self.path)+ path = unquote(parsed.path)+ if path not in XHR_RESPONSE_RESOURCE_PATHS:+ return False+ try:+ if path == "/xhr/resources/status.py":+ status, reason, content_type, body = _fetch_status_response(parsed.query)+ headers = [("X-Request-Method", self.command)]+ else:+ source = wpt_root / "xhr/resources/well-formed.xml"+ modified = formatdate(source.stat().st_mtime, usegmt=True)+ body = source.read_text(encoding="utf-8").encode("utf-8")+ status, reason, content_type = 200, None, "application/xml"+ headers = [("Last-Modified", modified)]+ except (ValueError, OSError, OverflowError):+ self.send_error(500)+ return True+ # These upstream handlers can respond without consuming the upload.+ # Close the connection so unread bytes cannot become another request.+ self.close_connection = True+ headers.append(("Connection", "close"))+ self._send_bytes(+ content_type, body, emit_body=emit_body, extra_headers=headers,+ status_code=status, status_text=reason,+ )+ return True+def _serve_fetch_abort_resource(self, path: str, query: str, *, emit_body: bool) -> None:diff --git a/moli-benchmark/tests/test_wpt_xhr_response_server.py b/moli-benchmark/tests/test_wpt_xhr_response_server.pynew file mode 100644index 000000000..c11ddd5f2--- /dev/null+++ b/moli-benchmark/tests/test_wpt_xhr_response_server.py@@ -0,0 +1,139 @@+from __future__ import annotations++import os+import tempfile+import unittest+from contextlib import ExitStack+from http.client import HTTPConnection+from pathlib import Path+from unittest.mock import patch++from moli_benchmark.wpt_cross.case_set import enumerate_cases+from moli_benchmark.wpt_cross.server import WptFixtureServer+++class XhrResponseFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ (self.root / "xhr/resources").mkdir(parents=True)+ self.xml = self.root / "xhr/resources/well-formed.xml"+ self.xml.write_bytes(b"<x>caf\xc3\xa9\r\n</x>")+ os.utime(self.xml, (946684800.875, 946684800.875))+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None+ ))++ def server(self) -> WptFixtureServer:+ return self.stack.enter_context(WptFixtureServer(self.root))++ def request(+ self, port: int, path: str, *, method: str = "GET",+ ) -> tuple[int, str, dict[str, str], bytes]:+ connection = HTTPConnection("127.0.0.1", port, timeout=2)+ try:+ connection.request(method, "/xhr/resources/" + path)+ response = connection.getresponse()+ return (+ response.status, response.reason,+ {name.lower(): value for name, value in response.getheaders()},+ response.read(),+ )+ finally:+ connection.close()++ def test_status_preserves_first_query_values_and_raw_response_bytes(self) -> None:+ server = self.server()+ cases = [+ ("", 200, "OMG", "", b""),+ ("code=402&code=200&text=caf%E9&text=ignored&type=text/xml&type=text/plain&content=%00%FF+%2B%25ff&content=ignored", 402, "caf\xe9", "text/xml", b"\x00\xff +%ff"),+ ("text=&type=&content=&content=ignored", 200, "", "", b""),+ ("code=699&text=custom&type=text/plain&content=unknown-status", 699, "custom", "text/plain", b"unknown-status"),+ ("type=application/xml;charset=windows-1252&content=%3Cx%3E%E6%A9%9F%3C%2Fx%3E", 200, "OMG", "application/xml;charset=windows-1252", b"<x>\xe6\xa9\x9f</x>"),+ ]+ for query, code, reason, content_type, expected in cases:+ with self.subTest(query=query):+ status, text, headers, body = self.request(server.port, "status.py?" + query)+ self.assertEqual((status, text, body), (code, reason, expected))+ self.assertEqual(headers["content-type"], content_type)+ self.assertEqual(headers["x-request-method"], "GET")+ self.assertEqual(headers["content-length"], str(len(expected)))+ for code in ("", "invalid"):+ with self.subTest(code=code):+ self.assertEqual(self.request(server.port, "status.py?code=" + code)[0], 500)++ def test_status_supports_custom_methods_and_head_preserves_response_metadata(self) -> None:+ server = self.server()+ for method in ("GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "CHICKEN"):+ with self.subTest(method=method):+ status, reason, headers, body = self.request(+ server.port, "status.py?code=402&text=FIVE+BUCKS&type=text/xml&content=%3Cx/%3E",+ method=method,+ )+ self.assertEqual((status, reason), (402, "FIVE BUCKS"))+ self.assertEqual(body, b"" if method == "HEAD" else b"<x/>")+ self.assertEqual(headers["content-length"], "4")+ self.assertEqual(headers["content-type"], "text/xml")+ self.assertEqual(headers["x-request-method"], method)++ def test_last_modified_uses_xml_file_contents_and_mtime_without_caching(self) -> None:+ server = self.server()+ for method in ("GET", "HEAD", "POST", "CHICKEN"):+ with self.subTest(method=method):+ status, _, headers, body = self.request(server.port, "last-modified.py", method=method)+ self.assertEqual(status, 200)+ self.assertEqual(body, b"" if method == "HEAD" else b"<x>caf\xc3\xa9\n</x>")+ self.assertEqual(headers["content-type"], "application/xml")+ self.assertEqual(headers["last-modified"], "Sat, 01 Jan 2000 00:00:00 GMT")+ self.assertEqual(headers["content-length"], str(len(b"<x>caf\xc3\xa9\n</x>")))+ self.xml.write_text("<updated/>")+ os.utime(self.xml, (946771200, 946771200))+ status, _, headers, body = self.request(server.port, "last-modified.py?ignored=1")+ self.assertEqual((status, body), (200, b"<updated/>"))+ self.assertEqual(headers["last-modified"], "Sun, 02 Jan 2000 00:00:00 GMT")+ self.xml.unlink()+ self.assertEqual(self.request(server.port, "last-modified.py")[0], 500)++ def test_response_fixtures_do_not_wait_for_unused_uploads(self) -> None:+ server = self.server()+ for path, expected in (("status.py?content=response", b"response"),+ ("last-modified.py", b"<x>caf\xc3\xa9\n</x>")):+ for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")):+ with self.subTest(path=path, framing=framing):+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.putrequest("POST", "/xhr/resources/" + path)+ connection.putheader(*framing)+ connection.endheaders()+ response = connection.getresponse()+ self.assertEqual((response.status, response.read()), (200, expected))+ self.assertEqual(response.headers["Connection"], "close")+ finally:+ connection.close()++ def test_case_selection_accepts_only_the_supported_response_fixture_paths(self) -> None:+ sources = {+ "xhr/status.window.js": "fetch('resources/status.py');",+ "xhr/metadata.html": "fetch('/xhr/resources/last-modified.py?ignored');",+ "xhr/nested/both.html": "fetch('../resources/status.py'); fetch('../resources/last-modified.py');",+ "xhr/unknown.html": "fetch('resources/status.py'); fetch('resources/unknown.py');",+ "xhr/suffix.html": "fetch('resources/status.py-extra');",+ "xhr/prefix.html": "fetch('/wrong/xhr/resources/last-modified.py');",+ "xhr/wrong-parent.html": "fetch('../resources/status.py');",+ }+ for path, source in sources.items():+ target = self.root / path+ target.parent.mkdir(parents=True, exist_ok=True)+ if path.endswith(".html"):+ source = (+ '<script src="/resources/testharness.js"></script>'+ f"<script>{source}</script>"+ )+ target.write_text(source)+ self.assertEqual([case.case_path for case in enumerate_cases(self.root, dir_prefixes=("xhr",))], [+ "xhr/metadata.html", "xhr/nested/both.html",+ "xhr/status.window.js?moli-wpt-script=window",+ ])