Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix open_in_browser ReDoS test flakiness and a broken exploit input (#8064)

Adrian2382636434ff
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 3324e9710..0ad9bd7be 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -216,8 +216,12 @@ def test_inject_base_url(body: bytes) -> None:
def _assert_open_in_browser_is_fast(body: bytes) -> None:
# The exploit inputs are large enough that a vulnerable implementation
# needs seconds to go through them, while a safe one stays in the low
- # milliseconds even on a slow interpreter.
- max_cpu_time = 0.2
+ # milliseconds even on a slow interpreter. The budget is set well above
+ # that low-milliseconds baseline to tolerate the syscall and cache/page-fault
+ # overhead that even a fast implementation picks up under CPU contention on
+ # busy CI runners, since process_time() counts that overhead even though it
+ # excludes time spent waiting for a CPU core.
+ max_cpu_time = 1
response = HtmlResponse("https://example.com", body=body)
start_time = process_time()
@@ -229,8 +233,11 @@ def _assert_open_in_browser_is_fast(body: bytes) -> None:
def test_open_in_browser_redos_comment():
# Exploit input from
# https://makenowjust-labs.github.io/recheck/playground/
- # for /<!--.*?-->/ (old pattern to remove comments).
- _assert_open_in_browser_is_fast(b"-><!--\x00" * 250_000 + b"->\n<!---->")
+ # for /<!--.*?-->/ (old pattern to remove comments). Unlike the linked
+ # exploit, this input has no "-->" anywhere, so a vulnerable
+ # implementation cannot short-circuit on an early match and degrades to
+ # quadratic behavior.
+ _assert_open_in_browser_is_fast(b"-><!--\x00" * 250_000)
def test_open_in_browser_redos_head():