Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix fragment inheritance for redirects with an empty fragment (#8100)
cui fliter0778a7411dac
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 49f1fa3e8..f7728d071 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, Self, cast-from urllib.parse import urljoin, urlparse+from urllib.parse import urljoinfrom w3lib.url import safe_url_string@@ -219,15 +219,17 @@ class RedirectMiddleware(BaseRedirectMiddleware):}:return response- assert response.headers["Location"] is not None- location = safe_url_string(response.headers["Location"])- if response.headers["Location"].startswith(b"//"):+ location_header = response.headers["Location"]+ assert location_header is not None+ location = safe_url_string(location_header)+ if location_header.startswith(b"//"):request_scheme = urlparse_cached(request).schemelocation = request_scheme + "://" + location.lstrip("/")redirected_url = urljoin(request.url, location)- if not urlparse(redirected_url).fragment:+ # A percent-encoded number sign is URI data, not a fragment delimiter.+ if b"#" not in location_header:fragment = urlparse_cached(request).fragmentif fragment:redirected_url = urljoin(redirected_url, f"#{fragment}")diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 180c1a669..770d7fea1 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -273,6 +273,26 @@ class TestRedirectMiddleware(TestRedirectBase):assert isinstance(req2, Request)assert req2.url == "http://www.example.com/redirected#target"+ def test_redirect_target_has_empty_fragment(self):+ url = "http://www.example.com/302#original"+ url2 = "http://www.example.com/redirected#"+ req = Request(url)+ rsp = Response(url, headers={"Location": url2}, status=302)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == "http://www.example.com/redirected"++ def test_redirect_target_has_percent_encoded_number_sign(self):+ url = "http://www.example.com/302#original"+ url2 = "http://www.example.com/redirected%23"+ req = Request(url)+ rsp = Response(url, headers={"Location": url2}, status=302)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == "http://www.example.com/redirected%23#original"+def test_redirect_302_head(self):url = "http://www.example.com/302"url2 = "http://www.example.com/redirected2"diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.pyindex 61e6d7c86..f67794ec8 100644--- a/tests/test_extension_debug.py+++ b/tests/test_extension_debug.py@@ -50,8 +50,8 @@ class SignalSpider(Spider):def test_stacktracedump_installs_signal_handlers() -> None:crawler = get_crawler()ext = build_from_crawler(StackTraceDump, crawler)- assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace # pylint: disable=comparison-with-callable- assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace # pylint: disable=comparison-with-callable+ assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace+ assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktracedef test_stacktracedump_works_without_signal_support(