Scrapy, a fast high-level web crawling & scraping framework for Python.
Add Response.follow_redirect() (#8331)
Adrian540dd13869d7
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 1fec3405c..bd302a42a 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -1312,6 +1312,8 @@ Response objects.. automethod:: Response.follow_all+ .. automethod:: Response.follow_redirect+.. automethod:: Response.to_dict.. automethod:: Response.from_dictdiff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex f7728d071..488a3c0ec 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -2,9 +2,6 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, Self, cast-from urllib.parse import urljoin--from w3lib.url import safe_url_stringfrom scrapy import signalsfrom scrapy.exceptions import IgnoreRequest, NotConfigured@@ -14,6 +11,7 @@ from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import global_object_namefrom scrapy.utils.response import get_meta_refresh+from scrapy.utils.url import _redirect_urlif TYPE_CHECKING:from scrapy import Request, Spider@@ -219,20 +217,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):}:return response- location_header = response.headers["Location"]- assert location_header is not None- location = safe_url_string(location_header)- if location_header.startswith(b"//"):- request_scheme = urlparse_cached(request).scheme- location = request_scheme + "://" + location.lstrip("/")-- redirected_url = urljoin(request.url, location)-- # A percent-encoded number sign is URI data, not a fragment delimiter.- if b"#" not in location_header:- fragment = urlparse_cached(request).fragment- if fragment:- redirected_url = urljoin(redirected_url, f"#{fragment}")+ assert response.headers["Location"] is not None+ redirected_url = _redirect_url(request.url, response.headers["Location"])redirected = self._build_redirect_request(request, response, url=redirected_url)if urlparse_cached(redirected).scheme not in {"http", "https"}:diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex c09a6773c..d1c6ae0d3 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -15,6 +15,7 @@ from scrapy.http.headers import Headersfrom scrapy.http.request import Requestfrom scrapy.link import Linkfrom scrapy.utils.trackref import object_ref+from scrapy.utils.url import _redirect_urlif TYPE_CHECKING:from collections.abc import Callable, Iterable, Mapping@@ -306,6 +307,30 @@ class Response(object_ref):parent_id=self.request.id if self.request else None,)+ def follow_redirect(self, **kwargs: Any) -> Request:+ """Return a :class:`~.Request` instance to follow the ``Location``+ header of this response.++ .. versionadded:: VERSION++ It accepts the same arguments as :meth:`follow`, except ``url``, which+ comes from the ``Location`` header. Unlike in :meth:`follow`, that URL is+ always resolved against :attr:`url`, never against the base URL of an+ HTML document, as the HTTP standard mandates for the ``Location``+ header.++ It raises :exc:`ValueError` if this response has no ``Location`` header.++ The resulting request uses the GET method and has no body unless you+ specify otherwise, even for 307 and 308 responses, for which+ :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`+ keeps the method and body of the original request.+ """+ location = self.headers.get("Location")+ if location is None:+ raise ValueError(f"{self} has no Location header")+ return self.follow(_redirect_url(self.url, location), **kwargs)+def follow_all(self,urls: Iterable[str | Link],diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex d5e2d700f..1292edde4 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -8,10 +8,10 @@ from __future__ import annotationsimport reimport warningsfrom typing import TYPE_CHECKING, TypeAlias-from urllib.parse import ParseResult, urlunparse+from urllib.parse import ParseResult, urljoin, urlunparseimport w3lib.url-from w3lib.url import any_to_uri, parse_url+from w3lib.url import any_to_uri, parse_url, safe_url_stringfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.httpobj import urlparse_cached@@ -188,3 +188,19 @@ def strip_url("" if strip_fragment else parsed_url.fragment,))+++def _redirect_url(url: str, location: bytes) -> str:+ """Return the absolute URL that the *location* value of the ``Location``+ header of a response to a request to *url* points to."""+ target = safe_url_string(location)+ parsed_url = urlparse_cached(url)+ if location.startswith(b"//"):+ # safe_url_string() may drop leading slashes of a scheme-relative URL,+ # so build the absolute URL without relying on urljoin().+ target = f"{parsed_url.scheme}://{target.lstrip('/')}"+ redirect_url = urljoin(url, target)+ # A percent-encoded number sign is URI data, not a fragment delimiter.+ if b"#" not in location and parsed_url.fragment:+ redirect_url = urljoin(redirect_url, f"#{parsed_url.fragment}")+ return redirect_urldiff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex 93b052a25..472180885 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -345,6 +345,18 @@ class TestTextResponse(TestResponseBase):absolute = "http://www.example.com/elsewhere/test"assert joined == absolute+ def test_follow_redirect_ignores_base_url(self):+ """The Location header is resolved against the response URL, not against+ the base URL of the document, which only applies to its own links."""+ body = b'<html><body><base href="https://example.net"></body></html>'+ response = self.response_class(+ "http://www.example.com/dir/index",+ body=body,+ headers={"Location": "/test"},+ )+ assert response.urljoin("/test") == "https://example.net/test"+ assert response.follow_redirect().url == "http://www.example.com/test"+def test_follow_selector(self):resp = self._links_response()urls = [diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.pyindex b299285c7..9df62a186 100644--- a/tests/utils/bases/http_response.py+++ b/tests/utils/bases/http_response.py@@ -279,6 +279,49 @@ class TestResponseBase(ABC):fol = res.follow("http://example.com/", flags=["cached", "allowed"])assert fol.flags == ["cached", "allowed"]+ # Response.follow_redirect++ @pytest.mark.parametrize(+ ("location", "target_url"),+ [+ ("http://foo.example.com/x", "http://foo.example.com/x"),+ ("/foo", "http://example.com/foo"),+ ("foo", "http://example.com/dir/foo"),+ (b"//foo.example.com/x", "http://foo.example.com/x"),+ (b"/a\xe7\xe3o", "http://example.com/a%E7%E3o"),+ ],+ )+ def test_follow_redirect(self, location, target_url):+ response = self.response_class(+ "http://example.com/dir/index", headers={"Location": location}+ )+ request = response.follow_redirect()+ assert request.url == target_url+ assert request.method == "GET"++ def test_follow_redirect_keeps_fragment(self):+ response = self.response_class(+ "http://example.com/index#frag", headers={"Location": "/foo"}+ )+ assert response.follow_redirect().url == "http://example.com/foo#frag"++ def test_follow_redirect_no_location(self):+ response = self.response_class("http://example.com")+ with pytest.raises(ValueError, match="has no Location header"):+ response.follow_redirect()++ def test_follow_redirect_kwargs(self):+ response = self.response_class(+ "http://example.com/index", headers={"Location": "/foo"}+ )+ request = response.follow_redirect(+ method="HEAD", meta={"foo": "bar"}, flags=["cached"]+ )+ assert request.url == "http://example.com/foo"+ assert request.method == "HEAD"+ assert request.meta["foo"] == "bar"+ assert request.flags == ["cached"]+def test_follow_parent_id(self):assert self.response_class("http://example.com").follow("foo").parent_id is Nonerequest = Request("http://example.com")