Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): serve the common HTML echo fixture
ldm0ebc016ace340
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex 4a5522a5a..6692f63a4 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -725,6 +725,19 @@ def _html_path_is_supported(return not any(token in rel for token in excluded)+@lru_cache(maxsize=None)+def _common_echo_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:+ resource = "common/echo.py"+ relative = posixpath.relpath(resource, directory)+ return tuple(+ re.compile(+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"+ )+ for reference in ("/" + resource, relative, "./" + relative)+ )++@lru_cache(maxsize=None)def _script_content_type_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:resource = "html/semantics/scripting-1/the-script-element/serve-with-content-type.py"@@ -809,6 +822,7 @@ def _supported_wptserve_handler_references(if rel is not None and rel.startswith("fetch/api/"):supported += _empty_location_handler_reference_patterns(posixpath.dirname(rel))if rel is not None:+ supported += _common_echo_handler_reference_patterns(posixpath.dirname(rel) or ".")supported += _navigation_handler_reference_patterns(posixpath.dirname(rel) or ".")supported += _json_module_handler_reference_patterns(posixpath.dirname(rel) or ".")if rel is not None:diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 5183ab57f..58681ba23 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -83,6 +83,7 @@ XHR_RESPONSE_RESOURCE_PATHS = {"/xhr/resources/status.py","/xhr/resources/last-modified.py",}+COMMON_ECHO_PATH = "/common/echo.py"FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py"FETCH_ABORT_RESOURCE_PATHS = {"/fetch/api/resources/stash-put.py",@@ -807,6 +808,7 @@ _REQUEST_TEMPLATE_RE = re.compile(_UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}")_ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}")_HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")+_TRICKLE_DELAY_RE = re.compile(r"d([0-9]+(?:\.[0-9]+)?)")_MAX_TRICKLE_DELAY_SECONDS = 10.0_LEGACY_WPT_RESOURCE_ALIASES = {"/resources/WebIDLParser.js": "resources/webidl2/lib/webidl2.js",@@ -1699,6 +1701,8 @@ def _make_handler(self._serve(emit_body=False)def do_OPTIONS(self) -> None: # noqa: N802+ if self._serve_common_echo_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif self._serve_xhr_response_resource():@@ -1731,6 +1735,8 @@ def _make_handler(self.send_error(404)def do_POST(self) -> None: # noqa: N802+ if self._serve_common_echo_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif self._serve_xhr_response_resource():@@ -1784,6 +1790,8 @@ def _make_handler(self.end_headers()def _serve_fetch_resource_method(self) -> None:+ if self._serve_common_echo_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif self._serve_xhr_response_resource():@@ -1815,6 +1823,8 @@ def _make_handler(do_DELETE = _serve_fetch_resource_methoddef do_YO(self) -> None: # noqa: N802 (WPT custom method)+ if self._serve_common_echo_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif unquote(urlparse(self.path).path) in {@@ -1950,6 +1960,81 @@ def _make_handler(except (TimeoutError, BrokenPipeError, ConnectionResetError, OSError):return+ def _substitute_response_template(+ self, body: bytes, path: str, query: str, *, escape_type: str,+ ) -> bytes:+ port = int(getattr(self.server, "wpt_primary_port", self.server.server_address[1]))+ alternate_port = int(getattr(self.server, "wpt_alternate_port", port))+ remote_port = int(getattr(self.server, "wpt_remote_port", alternate_port))+ hostname = _host_header_hostname(self.headers.get("Host"))+ return _substitute_wpt_template_variables(+ body,+ port=port,+ alternate_port=alternate_port,+ remote_port=remote_port,+ query=query,+ request_path=path,+ request_hostname=hostname,+ primary_hostname=str(getattr(self.server, "wpt_primary_hostname", hostname)),+ request_headers=self.headers,+ escape_type=escape_type,+ )++ def _serve_common_echo_resource(self) -> bool:+ parsed = urlsplit(self.path)+ if unquote(parsed.path) != COMMON_ECHO_PATH:+ return False+ # The handler only reads GET parameters, even for POST. Close the+ # connection instead of waiting for an unused request body.+ self.close_connection = True+ try:+ params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1")+ # wptserve's Request.GET preserves percent-decoded bytes and+ # MultiDict.first selects the first value, including an empty one.+ body = params["content"][0].encode("latin-1")+ headers = [("Content-Type", "text/html"), ("X-XSS-Protection", "0")]+ status, delay, auto_content_length = 200, 0.0, True+ for name, args in parse_pipe_commands(parsed.query):+ if name == "header":+ header_name, value = args[:2]+ value = value.replace("\r", " ").replace("\n", " ")+ if _valid_static_response_header(header_name, value):+ headers = _apply_header_operations(headers, [(+ header_name, value,+ len(args) == 3 and args[2].lower() in {"true", "1"},+ )])+ elif name == "status":+ status = int(args[0])+ elif name == "sub":+ body = self._substitute_response_template(+ body, COMMON_ECHO_PATH, parsed.query,+ escape_type=args[0] if args else "html",+ )+ elif name == "trickle":+ auto_content_length = False+ if not any(_headers_include(headers, name)+ for name in ("Cache-Control", "Pragma", "Expires")):+ headers.extend([+ ("Cache-Control", "no-cache, no-store, must-revalidate"),+ ("Pragma", "no-cache"),+ ("Expires", "0"),+ ])+ match = _TRICKLE_DELAY_RE.fullmatch(args[0])+ if match is not None:+ delay = max(delay, float(match.group(1)))+ except (KeyError, WptPipeError):+ self.send_error(500)+ return True+ if delay:+ time.sleep(min(delay, _MAX_TRICKLE_DELAY_SECONDS))+ self._send_bytes(+ None, body, emit_body=self.command != "HEAD",+ extra_headers=[*headers, ("Connection", "close")],+ status_code=status, cache_control=None,+ auto_content_length=auto_content_length,+ )+ return True+def _serve(self, *, emit_body: bool) -> None:try:self._serve_response(emit_body=emit_body)@@ -1957,6 +2042,8 @@ def _make_handler(self.send_error(500, "Invalid WPT template or pipe")def _serve_response(self, *, emit_body: bool) -> None:+ if self._serve_common_echo_resource():+ returnif self._serve_empty_location_resource(emit_body=emit_body):returnif self._serve_xhr_response_resource(emit_body=emit_body):@@ -2166,32 +2253,8 @@ def _make_handler()returnif _needs_wpt_template_substitution(file_path.name, body, parsed.query):- port = int(- getattr(- self.server,- "wpt_primary_port",- self.server.server_address[1],- )- )- alternate_port = int(getattr(self.server, "wpt_alternate_port", port))- remote_port = int(getattr(self.server, "wpt_remote_port", alternate_port))- primary_hostname = str(- getattr(- self.server,- "wpt_primary_hostname",- _host_header_hostname(self.headers.get("Host")),- )- )- body = _substitute_wpt_template_variables(- body,- port=port,- alternate_port=alternate_port,- remote_port=remote_port,- query=parsed.query,- request_path=path,- request_hostname=_host_header_hostname(self.headers.get("Host")),- primary_hostname=primary_hostname,- request_headers=self.headers,+ body = self._substitute_response_template(+ body, path, parsed.query,escape_type=_template_escape_type(file_path.name, parsed.query),)static_header_context = {@@ -2516,6 +2579,8 @@ def _make_handler()def __getattr__(self, name: str):+ if name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_ECHO_PATH:+ return self._serve_common_echo_resourceif name.startswith("do_") and unquote(urlparse(self.path).path) == NAVIGATION_SECOND_VISIT_PATH:return self._serve_navigation_second_visitif name.startswith("do_") and unquote(urlparse(self.path).path) == FETCH_EMPTY_LOCATION_PATH:@@ -2935,6 +3000,7 @@ def _make_handler(status_code: int = 200,status_text: str | None = None,cache_control: str | None = "no-store",+ auto_content_length: bool = True,) -> None:content_type, extra_headers = _response_content_type_and_extra_headers(content_type,@@ -2944,7 +3010,7 @@ def _make_handler(header_block = _static_response_header_block(content_type, extra_headers)for name, value in header_block:self.send_header(name, value)- if not _headers_include(header_block, "Content-Length"):+ if auto_content_length and not _headers_include(header_block, "Content-Length"):self.send_header("Content-Length", str(len(body)))if cache_control is not None:self.send_header("Cache-Control", cache_control)diff --git a/moli-benchmark/tests/test_wpt_common_echo_server.py b/moli-benchmark/tests/test_wpt_common_echo_server.pynew file mode 100644index 000000000..7675a167c--- /dev/null+++ b/moli-benchmark/tests/test_wpt_common_echo_server.py@@ -0,0 +1,170 @@+from __future__ import annotations++import tempfile+import unittest+from contextlib import ExitStack+from http.client import HTTPConnection+from pathlib import Path+from unittest.mock import patch+from urllib.parse import urlencode++from moli_benchmark.wpt_cross.case_set import enumerate_cases+from moli_benchmark.wpt_cross.server import WptFixtureServer+++ECHO_PATH = "/common/echo.py"+++class CommonEchoFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ (self.root / "common").mkdir()+ (self.root / "common/echo.py").write_text("# Python source must not be served")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None,+ ))+ self.server = self.stack.enter_context(WptFixtureServer(self.root))++ def request(self, query: str, *, method: str = "GET", path: str = ECHO_PATH, body=None):+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)+ try:+ connection.request(method, path + "?" + query, body, {"Origin": "https://caller.test"})+ response = connection.getresponse()+ return response.status, response.headers, response.read()+ finally:+ connection.close()++ def test_content_is_html_and_preserves_query_bytes_and_first_value(self) -> None:+ cases = (+ ("content=%3Cscript%3EglobalThis.answer%3D42%3C%2Fscript%3E", b"<script>globalThis.answer=42</script>"),+ ("content=first&content=second", b"first"),+ ("content=&content=second", b""),+ ("content", b""),+ ("content=A+B%2BC%26D%3DE", b"A B+C&D=E"),+ ("content=%FF%00%C3%A9%FE", b"\xff\x00\xc3\xa9\xfe"),+ ("co%6Etent=decoded-key", b"decoded-key"),+ ("content=%2520;%invalid", b"%20;%invalid"),+ ("content={{host}}", b"{{host}}"),+ )+ for query, expected in cases:+ for method in ("GET", "HEAD"):+ with self.subTest(query=query, method=method):+ status, headers, body = self.request(query, method=method)+ self.assertEqual(status, 200)+ self.assertEqual(headers.get_all("Content-Type"), ["text/html"])+ self.assertEqual(headers["X-XSS-Protection"], "0")+ self.assertEqual(headers["Content-Length"], str(len(expected)))+ self.assertEqual(body, b"" if method == "HEAD" else expected)+ self.assertIsNone(headers["Access-Control-Allow-Origin"])+ self.assertIsNone(headers["Cache-Control"])++ def test_all_methods_read_query_content_instead_of_request_body(self) -> None:+ for method in ("POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO", "chicken"):+ with self.subTest(method=method):+ status, headers, body = self.request(+ "content=query", method=method, body=b"content=upload",+ )+ self.assertEqual((status, body), (200, b"query"))+ self.assertEqual(headers["Content-Type"], "text/html")+ self.assertIsNone(headers["Access-Control-Allow-Origin"])++ def test_unused_uploads_do_not_delay_the_response(self) -> None:+ for method in ("POST", "PUT", "OPTIONS"):+ for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")):+ with self.subTest(method=method, framing=framing):+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)+ try:+ connection.putrequest(method, ECHO_PATH + "?content=ready")+ connection.putheader(*framing)+ connection.endheaders()+ response = connection.getresponse()+ self.assertEqual((response.status, response.read()), (200, b"ready"))+ self.assertEqual(response.headers["Connection"], "close")+ finally:+ connection.close()++ def test_missing_content_is_an_error_and_other_paths_are_not_echo_handlers(self) -> None:+ for method in ("GET", "HEAD", "POST"):+ for query in ("", "Content=wrong-case"):+ with self.subTest(method=method, query=query):+ self.assertEqual(self.request(query, method=method)[0], 500)+ for path in (ECHO_PATH + "2", "/wrong" + ECHO_PATH, "/COMMON/echo.py"):+ with self.subTest(path=path):+ self.assertEqual(self.request("content=not-an-echo", path=path)[0], 404)++ def test_response_pipes_apply_after_the_handler(self) -> None:+ query = urlencode({+ "content": "{{host}}",+ "pipe": "sub(none)|status(201)|header(Content-Type,text/plain)|header(X-XSS-Protection,1)",+ })+ for method in ("GET", "HEAD", "POST", "chicken"):+ with self.subTest(method=method):+ status, headers, body = self.request(query, method=method)+ self.assertEqual(status, 201)+ self.assertEqual(headers.get_all("Content-Type"), ["text/plain"])+ self.assertEqual(headers.get_all("X-XSS-Protection"), ["1"])+ self.assertEqual(headers["Content-Length"], "9")+ self.assertEqual(body, b"" if method == "HEAD" else b"localhost")+ with patch("moli_benchmark.wpt_cross.server.time.sleep") as sleep:+ status, headers, body = self.request(urlencode({"content": "delayed", "pipe": "trickle(d0.01)"}))+ self.assertEqual((status, body), (200, b"delayed"))+ self.assertIsNone(headers["Content-Length"])+ self.assertEqual(headers["Cache-Control"], "no-cache, no-store, must-revalidate")+ self.assertEqual(headers["Pragma"], "no-cache")+ self.assertEqual(headers["Expires"], "0")+ sleep.assert_called_once_with(0.01)+ for command, expected, pragma, expires in (+ ("trickle(d0)|header(Cache-Control,private,true)",+ ["no-cache, no-store, must-revalidate", "private"], "no-cache", "0"),+ ("header(Cache-Control,private)|trickle(d0)", ["private"], None, None),+ ("header(Pragma,no-cache)|trickle(d0)", None, "no-cache", None),+ ("header(Expires,0)|trickle(d0)", None, None, "0"),+ ):+ with self.subTest(command=command):+ status, headers, body = self.request(urlencode({"content": "ok", "pipe": command}))+ self.assertEqual((status, body), (200, b"ok"))+ self.assertEqual(headers.get_all("Cache-Control"), expected)+ self.assertEqual(headers["Pragma"], pragma)+ self.assertEqual(headers["Expires"], expires)+ for method in ("GET", "POST", "chicken"):+ self.assertEqual(self.request("content=not-served&pipe=unknown", method=method)[0], 500)++ def test_discovery_recognizes_only_references_to_the_common_echo_handler(self) -> None:+ directory = "html/browsers/echo-tests"+ cases = {+ "absolute.html": (ECHO_PATH, True),+ "relative.html": ("../../../common/echo.py", True),+ "dot-relative.html": ("./../../../common/echo.py", True),+ "bare.html": ("echo.py", False),+ "wrong-relative.html": ("../../common/echo.py", False),+ "wrong-root.html": ("/other/common/echo.py", False),+ "suffix.html": (ECHO_PATH + ".extra", False),+ }+ for name, (reference, _) in cases.items():+ path = self.root / directory / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(+ '<script src="/resources/testharness.js"></script>'+ f'<script>fetch("{reference}?content=hello")</script>'+ )+ (self.root / directory / "unknown.html").write_text(+ '<script src="/resources/testharness.js"></script>'+ f'<script>fetch("{ECHO_PATH}?content=x"); fetch("/unsupported.py")</script>'+ )+ (self.root / directory / "script.window.js").write_text(+ f'const echoURL = content => `{ECHO_PATH}?content=${{encodeURIComponent(content)}}`;'+ )+ discovered = enumerate_cases(self.root, dir_prefixes=(directory,))+ self.assertEqual(+ sorted(case.case_path.split("?")[0] for case in discovered),+ sorted([directory + "/" + name for name, (_, allowed) in cases.items() if allowed]+ + [directory + "/script.window.js"]),+ )+++if __name__ == "__main__":+ unittest.main()