Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix fragment inheritance for redirects with an empty fragment (#8100)

cui fliter0778a7411dac
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 49f1fa3e8..f7728d071 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Self, cast
-from urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin
from w3lib.url import safe_url_string
@@ -219,15 +219,17 @@ class RedirectMiddleware(BaseRedirectMiddleware):
}:
return response
- assert response.headers["Location"] is not None
- location = safe_url_string(response.headers["Location"])
- if response.headers["Location"].startswith(b"//"):
+ location_header = response.headers["Location"]
+ assert location_header is not None
+ location = safe_url_string(location_header)
+ if location_header.startswith(b"//"):
request_scheme = urlparse_cached(request).scheme
location = request_scheme + "://" + location.lstrip("/")
redirected_url = urljoin(request.url, location)
- if not urlparse(redirected_url).fragment:
+ # A percent-encoded number sign is URI data, not a fragment delimiter.
+ if b"#" not in location_header:
fragment = urlparse_cached(request).fragment
if fragment:
redirected_url = urljoin(redirected_url, f"#{fragment}")
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 180c1a669..770d7fea1 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -273,6 +273,26 @@ class TestRedirectMiddleware(TestRedirectBase):
assert isinstance(req2, Request)
assert req2.url == "http://www.example.com/redirected#target"
+ def test_redirect_target_has_empty_fragment(self):
+ url = "http://www.example.com/302#original"
+ url2 = "http://www.example.com/redirected#"
+ req = Request(url)
+ rsp = Response(url, headers={"Location": url2}, status=302)
+
+ req2 = self.mw.process_response(req, rsp)
+ assert isinstance(req2, Request)
+ assert req2.url == "http://www.example.com/redirected"
+
+ def test_redirect_target_has_percent_encoded_number_sign(self):
+ url = "http://www.example.com/302#original"
+ url2 = "http://www.example.com/redirected%23"
+ req = Request(url)
+ rsp = Response(url, headers={"Location": url2}, status=302)
+
+ req2 = self.mw.process_response(req, rsp)
+ assert isinstance(req2, Request)
+ assert req2.url == "http://www.example.com/redirected%23#original"
+
def test_redirect_302_head(self):
url = "http://www.example.com/302"
url2 = "http://www.example.com/redirected2"
diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.py
index 61e6d7c86..f67794ec8 100644
--- a/tests/test_extension_debug.py
+++ b/tests/test_extension_debug.py
@@ -50,8 +50,8 @@ class SignalSpider(Spider):
def test_stacktracedump_installs_signal_handlers() -> None:
crawler = get_crawler()
ext = build_from_crawler(StackTraceDump, crawler)
- assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace # pylint: disable=comparison-with-callable
- assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace # pylint: disable=comparison-with-callable
+ assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace
+ assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace
def test_stacktracedump_works_without_signal_support(