Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(wpt): serve XHR request URI and redirect fixtures

ldm07ec4a3ac15fe
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
index 36ceaa993..ac30d06a9 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
@@ -833,6 +833,24 @@ def _remote_context_handler_reference_patterns(directory: str) -> tuple[re.Patte
)
+
+@lru_cache(maxsize=None)
+def _xhr_url_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:
+ references = []
+ for name in ("requri.py", "redirect.py"):
+ resource = f"xhr/resources/{name}"
+ relative = posixpath.relpath(resource, directory)
+ references.extend(("/" + resource, relative, "./" + relative))
+ return tuple(
+ re.compile(
+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"
+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"
+ )
+ for reference in references
+ )
+
+
+
def _supported_wptserve_handler_references(
rel: str | None,
) -> tuple[re.Pattern[str], ...]:
@@ -862,6 +880,7 @@ def _supported_wptserve_handler_references(
supported += _script_load_error_handler_reference_patterns(posixpath.dirname(rel) or ".")
if rel is not None and rel.startswith("xhr/"):
supported += SUPPORTED_XHR_DELAY_WPTSERVE_HANDLER_PATTERNS
+ supported += _xhr_url_handler_reference_patterns(rel.rsplit("/", 1)[0])
if rel is not None and rel.startswith(
"html/semantics/scripting-1/the-script-element/module/"
):
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 2c1fe7c06..f6b20cee5 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -130,6 +130,10 @@ IFRAME_STASH_PATH = "/html/semantics/embedded-content/the-iframe-element/stash.p
COMMON_ECHO_PATH = "/common/echo.py"
COMMON_REDIRECT_PATH = "/common/redirect.py"
FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py"
+XHR_URL_RESOURCE_PATHS = {
+ "/xhr/resources/requri.py",
+ "/xhr/resources/redirect.py",
+}
FETCH_ABORT_RESOURCE_PATHS = {
"/fetch/api/resources/stash-put.py",
"/fetch/api/resources/stash-take.py",
@@ -1261,6 +1265,32 @@ def _form_submission_response(
return b"OK" if valid else b"FAIL"
+def _xhr_redirect_fixture_response(
+ path: str, query: str
+) -> tuple[int, str | None, list[tuple[str, str]], bytes, float | None]:
+ """Model xhr/resources/redirect.py, including its second Location decode."""
+ params = parse_qs(query, keep_blank_values=True, encoding="latin-1")
+ code = int(params.get("code", ["302"])[0])
+ location = params.get("location", [path + "?followed"])[0]
+ location = location.encode("latin-1").decode("utf-8")
+ if location:
+ location = parse_qs("location=" + location)["location"][0]
+ if location.startswith("redirect.py"):
+ location += "&code=" + str(code)
+ delay = None
+ if "delay" in params:
+ delay = float(params["delay"][0]) / 1_000
+ if not math.isfinite(delay) or delay < 0:
+ raise ValueError("invalid redirect delay")
+ if "followed" in params:
+ # Preserve the upstream handler's header spelling.
+ return 200, None, [("Content:Type", "text/plain")], b"MAGIC HAPPENED", delay
+ if any(character in location for character in "\r\n"):
+ raise ValueError("invalid Location header")
+ location.encode("latin-1")
+ return code, "WEBSRT MARKETING", [("Location", location)], b"TEST", delay
+
+
def _redirect_fixture_response(query: str) -> tuple[int, str] | None:
"""Return the shared redirect response used by static WPT fixture handlers."""
@@ -2048,6 +2078,9 @@ requestExecutor("{executor_uuid}", {start_on_js});
return True
+ def _serve_xhr_url_method(self) -> None:
+ self._serve_xhr_url_resource(emit_body=self.command != "HEAD")
+
def do_GET(self) -> None: # noqa: N802 (BaseHTTPRequestHandler API)
if self.headers.get("Upgrade", "").lower() == "websocket":
self._serve_websocket()
@@ -2081,7 +2114,10 @@ requestExecutor("{executor_uuid}", {start_on_js});
unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",
)
return
- if self._serve_xhr_response_resource():
+ if (
+ self._serve_xhr_response_resource()
+ or self._serve_xhr_url_resource(emit_body=True)
+ ):
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
@@ -2134,7 +2170,10 @@ requestExecutor("{executor_uuid}", {start_on_js});
unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",
)
return
- if self._serve_xhr_response_resource():
+ if (
+ self._serve_xhr_response_resource()
+ or self._serve_xhr_url_resource(emit_body=True)
+ ):
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
@@ -2219,7 +2258,10 @@ requestExecutor("{executor_uuid}", {start_on_js});
unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",
)
return
- if self._serve_xhr_response_resource():
+ if (
+ self._serve_xhr_response_resource()
+ or self._serve_xhr_url_resource(emit_body=True)
+ ):
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
@@ -2278,6 +2320,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",
)
return
+ if self._serve_xhr_url_resource(emit_body=True):
+ return
parsed = urlparse(self.path)
if unquote(parsed.path) == NAVIGATION_SECOND_VISIT_PATH:
self._serve_navigation_second_visit()
@@ -2477,7 +2521,10 @@ requestExecutor("{executor_uuid}", {start_on_js});
unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD",
)
return
- if self._serve_xhr_response_resource(emit_body=emit_body):
+ if (
+ self._serve_xhr_response_resource(emit_body=emit_body)
+ or self._serve_xhr_url_resource(emit_body=emit_body)
+ ):
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
@@ -3216,6 +3263,52 @@ requestExecutor("{executor_uuid}", {start_on_js});
self.send_error(500)
+ def _serve_xhr_url_resource(self, *, emit_body: bool) -> bool:
+ parsed = urlsplit(self.path)
+ path = unquote(parsed.path)
+ if path not in XHR_URL_RESOURCE_PATHS:
+ return False
+ try:
+ if path == "/xhr/resources/requri.py":
+ params = parse_qs(parsed.query, keep_blank_values=True)
+ uri = self.path
+ if "full" in params and not uri.startswith("http://"):
+ authority = self.headers.get("Host")
+ if authority is None:
+ authority = _url_host_literal(
+ str(self.server.server_address[0])
+ )
+ if urlsplit("//" + authority).port is None:
+ authority += ":" + str(self.server.server_address[1])
+ uri = f"http://{authority}{uri}"
+ status, reason, headers, body = 200, None, [], uri.encode("utf-8")
+ else:
+ status, reason, headers, body, delay = _xhr_redirect_fixture_response(
+ parsed.path, parsed.query
+ )
+ if delay is not None:
+ time.sleep(delay)
+ except (ValueError, KeyError, OverflowError):
+ self.send_error(500)
+ return True
+ # These handlers reply without reading uploads. Close connections
+ # with unread bodies so a redirect can arrive before upload finishes.
+ if (
+ self.headers.get("Transfer-Encoding") is not None
+ or self.headers.get("Content-Length", "0").strip() not in {"", "0"}
+ ):
+ self.close_connection = True
+ headers.append(("Connection", "close"))
+ self._send_bytes(
+ None,
+ body,
+ emit_body=emit_body,
+ extra_headers=headers,
+ status_code=status,
+ status_text=reason,
+ )
+ return True
+
def _serve_xhr_delay(self, query: str, *, emit_body: bool) -> None:
delay_seconds = _wpt_delay_seconds(query)
if delay_seconds is None:
@@ -3269,6 +3362,9 @@ requestExecutor("{executor_uuid}", {start_on_js});
return self._serve_document_charset_resource
if name.startswith("do_") and unquote(urlsplit(self.path).path) == IFRAME_STASH_PATH:
return self._serve_iframe_stash_resource
+
+ if name.startswith("do_") and unquote(urlsplit(self.path).path) in XHR_URL_RESOURCE_PATHS:
+ return self._serve_xhr_url_method
if name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_ECHO_PATH:
return self._serve_common_echo_resource
if name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_REDIRECT_PATH:
@@ -3908,12 +4004,15 @@ requestExecutor("{executor_uuid}", {start_on_js});
cache_control: str | None = "no-store",
auto_content_length: bool = True,
) -> None:
- content_type, extra_headers = _response_content_type_and_extra_headers(
- content_type,
- extra_headers,
- )
+ if content_type is None:
+ header_block = list(extra_headers or [])
+ else:
+ content_type, extra_headers = _response_content_type_and_extra_headers(
+ content_type,
+ extra_headers,
+ )
+ header_block = _static_response_header_block(content_type, extra_headers)
self.send_response(status_code, status_text)
- header_block = _static_response_header_block(content_type, extra_headers)
for name, value in header_block:
self.send_header(name, value)
if auto_content_length and not _headers_include(header_block, "Content-Length"):
diff --git a/moli-benchmark/tests/test_wpt_xhr_url_server.py b/moli-benchmark/tests/test_wpt_xhr_url_server.py
new file mode 100644
index 000000000..ad4bc2428
--- /dev/null
+++ b/moli-benchmark/tests/test_wpt_xhr_url_server.py
@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from contextlib import ExitStack
+from http.client import HTTPConnection
+from pathlib import Path
+from unittest.mock import patch
+from urllib.parse import urlencode
+
+from moli_benchmark.wpt_cross.case_set import enumerate_cases
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+RESOURCE_DIR = "/xhr/resources/"
+
+
+class XhrUrlFixtureTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+ (self.root / "resources").mkdir()
+ (self.root / "resources/testharness.js").write_text("// testharness")
+ (self.root / "resources/testharnessreport.js").write_text("// report")
+ self.stack.enter_context(patch(
+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None
+ ))
+
+ def server(self, *, keep_alive: bool = False) -> WptFixtureServer:
+ server = WptFixtureServer(self.root)
+ if keep_alive:
+ server.httpd.RequestHandlerClass.protocol_version = "HTTP/1.1"
+ return self.stack.enter_context(server)
+
+ def request(
+ self, port: int, target: str, *, method: str = "GET",
+ headers: dict[str, str] | None = None,
+ ) -> tuple[int, str, dict[str, str], bytes]:
+ connection = HTTPConnection("127.0.0.1", port, timeout=2)
+ try:
+ connection.request(method, target, headers=headers or {})
+ response = connection.getresponse()
+ return (
+ response.status, response.reason,
+ {name.lower(): value for name, value in response.getheaders()},
+ response.read(),
+ )
+ finally:
+ connection.close()
+
+ def test_request_uri_preserves_raw_path_and_query(self) -> None:
+ server = self.server()
+ for path in (
+ "requri.py", "requri.py?", "requri.py?help=",
+ "requri.py?%23foobar", "%72equri.py?a=%2f%23&b=+%20",
+ ):
+ with self.subTest(path=path):
+ target = RESOURCE_DIR + path
+ status, _, headers, body = self.request(server.port, target)
+ self.assertEqual((status, body), (200, target.encode()))
+ self.assertNotIn("content-type", headers)
+
+ for query in ("full", "full=", "full=0", "full&full=ignored"):
+ with self.subTest(query=query):
+ target = RESOURCE_DIR + "requri.py?" + query
+ for host, authority in (
+ ("example.test:1234", "example.test:1234"),
+ ("example.test", f"example.test:{server.port}"),
+ ):
+ status, _, _, body = self.request(
+ server.port, target, headers={"Host": host}
+ )
+ self.assertEqual(
+ (status, body), (200, f"http://{authority}{target}".encode())
+ )
+
+ target = "http://example.test:1234" + RESOURCE_DIR + "requri.py?full"
+ status, _, _, body = self.request(server.port, target)
+ self.assertEqual((status, body), (200, target.encode()))
+
+ def test_fixtures_accept_custom_methods_and_head_omits_only_body(self) -> None:
+ server = self.server()
+ for method in ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "YO", "CHICKEN"):
+ for resource, status, expected in (
+ ("requri.py?x=%23", 200, b"/xhr/resources/requri.py?x=%23"),
+ ("redirect.py", 302, b"TEST"),
+ ):
+ with self.subTest(method=method, resource=resource):
+ actual, _, headers, body = self.request(
+ server.port, RESOURCE_DIR + resource, method=method
+ )
+ self.assertEqual(actual, status)
+ self.assertEqual(body, b"" if method == "HEAD" else expected)
+ self.assertEqual(headers["content-length"], str(len(expected)))
+ self.assertNotIn("content-type", headers)
+ status, _, _, _ = self.request(server.port, "/unhandled.py", method="CHICKEN")
+ self.assertEqual(status, 501)
+
+ def test_redirect_status_location_and_first_query_values(self) -> None:
+ server = self.server()
+ cases = [
+ ("", 302, RESOURCE_DIR + "redirect.py?followed"),
+ ("code=307&code=301&location=first&location=second", 307, "first"),
+ ("location=%252Ftarget%253Fx%253D1%2523fragment", 302, "/target?x=1#fragment"),
+ ("location=redirect.py%3Flocation%3Dcontent.py&code=308", 308,
+ "redirect.py?location=content.py&code=308"),
+ ("location=", 302, ""),
+ (urlencode({"location": "javascript:invalid"}), 302, "javascript:invalid"),
+ (urlencode({"location": "http://example.test/target?q=1"}), 302,
+ "http://example.test/target?q=1"),
+ ]
+ cases.extend((f"code={code}", code, RESOURCE_DIR + "redirect.py?followed")
+ for code in (200, 300, 301, 303, 399))
+ for query, code, location in cases:
+ with self.subTest(query=query):
+ status, reason, headers, body = self.request(
+ server.port, RESOURCE_DIR + "redirect.py?" + query
+ )
+ self.assertEqual((status, reason, body), (code, "WEBSRT MARKETING", b"TEST"))
+ self.assertEqual(headers["location"], location)
+ self.assertNotIn("content-type", headers)
+
+ def test_redirect_followed_response_and_delay_match_upstream(self) -> None:
+ server = self.server()
+ _, _, redirect_headers, _ = self.request(server.port, RESOURCE_DIR + "redirect.py")
+ status, _, headers, body = self.request(server.port, redirect_headers["location"])
+ self.assertEqual((status, body), (200, b"MAGIC HAPPENED"))
+ # Upstream spells this header "Content:Type"; it is not Content-Type.
+ self.assertEqual(headers["content"], "Type: text/plain")
+ self.assertNotIn("content-type", headers)
+ self.assertNotIn("location", headers)
+ # Once followed, the parsed location is no longer used as a header.
+ status, _, headers, body = self.request(
+ server.port, RESOURCE_DIR + "redirect.py?followed&location=%E2%82%AC"
+ )
+ self.assertEqual((status, body), (200, b"MAGIC HAPPENED"))
+ self.assertNotIn("location", headers)
+ with patch("moli_benchmark.wpt_cross.server.time.sleep") as sleep:
+ for query, method in (
+ ("delay=250&delay=1000", "GET"),
+ ("delay=250&followed=", "GET"),
+ ("delay=250", "HEAD"),
+ ):
+ self.request(server.port, RESOURCE_DIR + "redirect.py?" + query, method=method)
+ sleep.assert_called_once_with(0.25)
+ sleep.reset_mock()
+ for query in (
+ "code=invalid", "code=", "delay=invalid", "delay=nan", "delay=inf",
+ "delay=-1", "location=%FF", "location=%250D%250A",
+ "code=invalid&followed",
+ ):
+ with self.subTest(query=query):
+ status, _, _, _ = self.request(server.port, RESOURCE_DIR + "redirect.py?" + query)
+ self.assertEqual(status, 500)
+
+ def test_response_can_arrive_before_upload_body(self) -> None:
+ server = self.server(keep_alive=True)
+ for resource, expected in (
+ ("redirect.py", (302, b"TEST")),
+ ("requri.py", (200, b"/xhr/resources/requri.py")),
+ ):
+ for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")):
+ with self.subTest(resource=resource, framing=framing):
+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)
+ try:
+ connection.putrequest("POST", RESOURCE_DIR + resource)
+ connection.putheader(*framing)
+ connection.endheaders()
+ # No upload bytes have been sent when we read the response.
+ response = connection.getresponse()
+ self.assertEqual((response.status, response.read()), expected)
+ self.assertEqual(response.headers["Connection"], "close")
+ self.assertTrue(response.will_close)
+ finally:
+ connection.close()
+
+ def test_bodyless_requests_can_reuse_connection(self) -> None:
+ server = self.server(keep_alive=True)
+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)
+ try:
+ connection.request("GET", RESOURCE_DIR + "redirect.py")
+ redirect = connection.getresponse()
+ self.assertEqual((redirect.status, redirect.read()), (302, b"TEST"))
+ first_socket = connection.sock
+ self.assertIsNotNone(first_socket)
+ connection.request("CHICKEN", redirect.headers["Location"])
+ followed = connection.getresponse()
+ self.assertEqual((followed.status, followed.read()), (200, b"MAGIC HAPPENED"))
+ self.assertIs(connection.sock, first_socket)
+ finally:
+ connection.close()
+
+ def test_case_selection_resolves_only_supported_xhr_resource_paths(self) -> None:
+ sources = {
+ "xhr/absolute.html": "fetch('/xhr/resources/requri.py?full');",
+ "xhr/relative.html": "fetch('resources/requri.py'); fetch('./resources/redirect.py');",
+ "xhr/nested/parent.html": "fetch('../resources/redirect.py');",
+ "xhr/unsupported.html": "fetch('resources/requri.py'); fetch('resources/other.py');",
+ "xhr/suffix.html": "fetch('resources/redirect.py-extra');",
+ "xhr/script-suffix.html": "fetch('resources/redirect.py.js');",
+ "xhr/prefix.html": "fetch('/wrong/xhr/resources/redirect.py');",
+ "xhr/wrong-parent.html": "fetch('../resources/redirect.py');",
+ "xhr/wrong-root.html": "fetch('redirect.py');",
+ "xhr/nested/wrong-relative.html": "fetch('resources/redirect.py');",
+ }
+ for path, script in sources.items():
+ target = self.root / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(
+ '<script src="/resources/testharness.js"></script>'
+ '<script src="/resources/testharnessreport.js"></script>'
+ f"<script>{script}</script>"
+ )
+ selected = enumerate_cases(self.root, dir_prefixes=("xhr",))
+ self.assertEqual([case.case_path for case in selected], [
+ "xhr/absolute.html", "xhr/nested/parent.html", "xhr/relative.html",
+ ])