Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add Response.follow_redirect() (#8331)

Adrian540dd13869d7
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1fec3405c..bd302a42a 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1312,6 +1312,8 @@ Response objects
.. automethod:: Response.follow_all
+ .. automethod:: Response.follow_redirect
+
.. automethod:: Response.to_dict
.. automethod:: Response.from_dict
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index f7728d071..488a3c0ec 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -2,9 +2,6 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Self, cast
-from urllib.parse import urljoin
-
-from w3lib.url import safe_url_string
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
@@ -14,6 +11,7 @@ from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import global_object_name
from scrapy.utils.response import get_meta_refresh
+from scrapy.utils.url import _redirect_url
if TYPE_CHECKING:
from scrapy import Request, Spider
@@ -219,20 +217,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
}:
return response
- location_header = response.headers["Location"]
- assert location_header is not None
- location = safe_url_string(location_header)
- if location_header.startswith(b"//"):
- request_scheme = urlparse_cached(request).scheme
- location = request_scheme + "://" + location.lstrip("/")
-
- redirected_url = urljoin(request.url, location)
-
- # A percent-encoded number sign is URI data, not a fragment delimiter.
- if b"#" not in location_header:
- fragment = urlparse_cached(request).fragment
- if fragment:
- redirected_url = urljoin(redirected_url, f"#{fragment}")
+ assert response.headers["Location"] is not None
+ redirected_url = _redirect_url(request.url, response.headers["Location"])
redirected = self._build_redirect_request(request, response, url=redirected_url)
if urlparse_cached(redirected).scheme not in {"http", "https"}:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index c09a6773c..d1c6ae0d3 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -15,6 +15,7 @@ from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
+from scrapy.utils.url import _redirect_url
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Mapping
@@ -306,6 +307,30 @@ class Response(object_ref):
parent_id=self.request.id if self.request else None,
)
+ def follow_redirect(self, **kwargs: Any) -> Request:
+ """Return a :class:`~.Request` instance to follow the ``Location``
+ header of this response.
+
+ .. versionadded:: VERSION
+
+ It accepts the same arguments as :meth:`follow`, except ``url``, which
+ comes from the ``Location`` header. Unlike in :meth:`follow`, that URL is
+ always resolved against :attr:`url`, never against the base URL of an
+ HTML document, as the HTTP standard mandates for the ``Location``
+ header.
+
+ It raises :exc:`ValueError` if this response has no ``Location`` header.
+
+ The resulting request uses the GET method and has no body unless you
+ specify otherwise, even for 307 and 308 responses, for which
+ :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
+ keeps the method and body of the original request.
+ """
+ location = self.headers.get("Location")
+ if location is None:
+ raise ValueError(f"{self} has no Location header")
+ return self.follow(_redirect_url(self.url, location), **kwargs)
+
def follow_all(
self,
urls: Iterable[str | Link],
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index d5e2d700f..1292edde4 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -8,10 +8,10 @@ from __future__ import annotations
import re
import warnings
from typing import TYPE_CHECKING, TypeAlias
-from urllib.parse import ParseResult, urlunparse
+from urllib.parse import ParseResult, urljoin, urlunparse
import w3lib.url
-from w3lib.url import any_to_uri, parse_url
+from w3lib.url import any_to_uri, parse_url, safe_url_string
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
@@ -188,3 +188,19 @@ def strip_url(
"" if strip_fragment else parsed_url.fragment,
)
)
+
+
+def _redirect_url(url: str, location: bytes) -> str:
+ """Return the absolute URL that the *location* value of the ``Location``
+ header of a response to a request to *url* points to."""
+ target = safe_url_string(location)
+ parsed_url = urlparse_cached(url)
+ if location.startswith(b"//"):
+ # safe_url_string() may drop leading slashes of a scheme-relative URL,
+ # so build the absolute URL without relying on urljoin().
+ target = f"{parsed_url.scheme}://{target.lstrip('/')}"
+ redirect_url = urljoin(url, target)
+ # A percent-encoded number sign is URI data, not a fragment delimiter.
+ if b"#" not in location and parsed_url.fragment:
+ redirect_url = urljoin(redirect_url, f"#{parsed_url.fragment}")
+ return redirect_url
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 93b052a25..472180885 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -345,6 +345,18 @@ class TestTextResponse(TestResponseBase):
absolute = "http://www.example.com/elsewhere/test"
assert joined == absolute
+ def test_follow_redirect_ignores_base_url(self):
+ """The Location header is resolved against the response URL, not against
+ the base URL of the document, which only applies to its own links."""
+ body = b'<html><body><base href="https://example.net"></body></html>'
+ response = self.response_class(
+ "http://www.example.com/dir/index",
+ body=body,
+ headers={"Location": "/test"},
+ )
+ assert response.urljoin("/test") == "https://example.net/test"
+ assert response.follow_redirect().url == "http://www.example.com/test"
+
def test_follow_selector(self):
resp = self._links_response()
urls = [
diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py
index b299285c7..9df62a186 100644
--- a/tests/utils/bases/http_response.py
+++ b/tests/utils/bases/http_response.py
@@ -279,6 +279,49 @@ class TestResponseBase(ABC):
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
assert fol.flags == ["cached", "allowed"]
+ # Response.follow_redirect
+
+ @pytest.mark.parametrize(
+ ("location", "target_url"),
+ [
+ ("http://foo.example.com/x", "http://foo.example.com/x"),
+ ("/foo", "http://example.com/foo"),
+ ("foo", "http://example.com/dir/foo"),
+ (b"//foo.example.com/x", "http://foo.example.com/x"),
+ (b"/a\xe7\xe3o", "http://example.com/a%E7%E3o"),
+ ],
+ )
+ def test_follow_redirect(self, location, target_url):
+ response = self.response_class(
+ "http://example.com/dir/index", headers={"Location": location}
+ )
+ request = response.follow_redirect()
+ assert request.url == target_url
+ assert request.method == "GET"
+
+ def test_follow_redirect_keeps_fragment(self):
+ response = self.response_class(
+ "http://example.com/index#frag", headers={"Location": "/foo"}
+ )
+ assert response.follow_redirect().url == "http://example.com/foo#frag"
+
+ def test_follow_redirect_no_location(self):
+ response = self.response_class("http://example.com")
+ with pytest.raises(ValueError, match="has no Location header"):
+ response.follow_redirect()
+
+ def test_follow_redirect_kwargs(self):
+ response = self.response_class(
+ "http://example.com/index", headers={"Location": "/foo"}
+ )
+ request = response.follow_redirect(
+ method="HEAD", meta={"foo": "bar"}, flags=["cached"]
+ )
+ assert request.url == "http://example.com/foo"
+ assert request.method == "HEAD"
+ assert request.meta["foo"] == "bar"
+ assert request.flags == ["cached"]
+
def test_follow_parent_id(self):
assert self.response_class("http://example.com").follow("foo").parent_id is None
request = Request("http://example.com")