Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove the non-standard 307/308 handling, and align other aspects with the standard
Adrian Chavesba3d7bc7a832
diff --git a/docs/news.rst b/docs/news.rstindex d8efa9ba0..db6749ae7 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -8,17 +8,55 @@ Release notesScrapy 2.14.2 (unreleased)---------------------------Security bug fixes-~~~~~~~~~~~~~~~~~~+Deprecations+~~~~~~~~~~~~++- Using a response URL string as the first parameter on calls to+ :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is+ deprecated. Pass a :class:`~scrapy.http.Response` instead.++ The parameter has also been renamed to ``response`` to reflect this change.+ The old parameter name (``resp_or_url``) is deprecated.++Bug fixes+~~~~~~~~~++- Aligned redirect method conversions to ``GET`` with the `standard+ <https://fetch.spec.whatwg.org/#http-redirect-fetch>`__:++ - 301 redirects of ``POST`` requests turn into ``GET`` requests.++ - Only ``POST`` 302 redirects turn into ``GET`` requests, other methods+ are preserved.++ - ``HEAD`` 303 redirects do not turn into ``GET`` requests.++ - ``GET`` 303 redirects do not get their body or standard ``Content-*``+ headers removed.++ .. note:: Turning into a ``GET`` request implies not only a method change,+ but also omitting the body and ``Content-*`` headers in the redirect+ request.++- Redirects where the original request body is dropped now also get their+ ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers+ removed, in addition to ``Content-Type`` and ``Content-Length`` that were+ already being removed.++- Redirects now maintain the source URL fragment if the redirect URL doesn't+ have one. This may be useful when using browser-based download handlers,+ like those of `scrapy-playwright`_ or `scrapy-zyte-api`_, and letting+ Scrapy handle redirects.-- Cross-origin 307/308 redirects now remove the request body. See the- 7j88-353p-36rg_ security advisory for details.+ .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright+ .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/- .. _7j88-353p-36rg: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j88-353p-36rg+- The ``Referer`` header is now removed on redirect if+ :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled.-- 301 redirects now force the request method to GET, in line with modern- browsers. The original request method is now only maintained on 307 and 308- redirects.+- The handling of the ``Referer`` header on redirects now accounts for the+ ``Referer-Policy`` header of the response that triggers the redirect+ request... _release-2.14.1:diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 0ab2f58bc..9520a5cbb 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -2,14 +2,17 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, cast-from urllib.parse import urljoin+from urllib.parse import urljoin, urlparsefrom w3lib.url import safe_url_string+from scrapy import signalsfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import HtmlResponse, Response+from scrapy.spidermiddlewares.referer import RefererMiddlewarefrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cached+from scrapy.utils.python import global_object_namefrom scrapy.utils.response import get_meta_refreshif TYPE_CHECKING:@@ -24,67 +27,6 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-def _build_redirect_request(- source_request: Request, *, url: str, **kwargs: Any-) -> Request:- redirect_request = source_request.replace(- url=url,- **kwargs,- cls=None,- cookies=None,- )- if "_scheme_proxy" in redirect_request.meta:- source_request_scheme = urlparse_cached(source_request).scheme- redirect_request_scheme = urlparse_cached(redirect_request).scheme- if source_request_scheme != redirect_request_scheme:- redirect_request.meta.pop("_scheme_proxy")- redirect_request.meta.pop("proxy", None)- redirect_request.meta.pop("_auth_proxy", None)- redirect_request.headers.pop(b"Proxy-Authorization", None)- has_cookie_header = "Cookie" in redirect_request.headers- has_authorization_header = "Authorization" in redirect_request.headers- has_request_body = bool(getattr(redirect_request, "body", None))- if has_cookie_header or has_authorization_header or has_request_body:- default_ports = {"http": 80, "https": 443}-- parsed_source_request = urlparse_cached(source_request)- source_scheme, source_host, source_port = (- parsed_source_request.scheme,- parsed_source_request.hostname,- parsed_source_request.port- or default_ports.get(parsed_source_request.scheme),- )-- parsed_redirect_request = urlparse_cached(redirect_request)- redirect_scheme, redirect_host, redirect_port = (- parsed_redirect_request.scheme,- parsed_redirect_request.hostname,- parsed_redirect_request.port- or default_ports.get(parsed_redirect_request.scheme),- )-- if has_cookie_header and (- redirect_scheme not in {source_scheme, "https"}- or source_host != redirect_host- ):- del redirect_request.headers["Cookie"]-- # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name- if has_authorization_header and (- source_scheme != redirect_scheme- or source_host != redirect_host- or source_port != redirect_port- ):- del redirect_request.headers["Authorization"]-- if has_request_body and source_host != redirect_host:- redirect_request = redirect_request.replace(body=b"")- redirect_request.headers.pop("Content-Type", None)- redirect_request.headers.pop("Content-Length", None)-- return redirect_request--class BaseRedirectMiddleware:crawler: Crawlerenabled_setting: str = "REDIRECT_ENABLED"@@ -95,13 +37,59 @@ class BaseRedirectMiddleware:self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES")self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST")+ self._referer_spider_middleware: RefererMiddleware | None = None@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:o = cls(crawler.settings)o.crawler = crawler+ crawler.signals.connect(o._engine_started, signal=signals.engine_started)return o+ def handle_referer(self, request: Request, response: Response) -> None:+ """Remove, modify or keep the Referer header of *request* based on the+ *response* that triggered *request*.++ By default, this method finds a run-time instance of+ scrapy.spidermiddlewares.referer.RefererMiddleware (or of a subclass)+ and uses it to set the right Referer header.++ Override this method if you use a different Scrapy component to handle+ Referer headers, of if you want to use a custom logic to set the+ Referer header on redirects.+ """+ request.headers.pop("Referer", None)+ if not self._referer_spider_middleware:+ return+ self._referer_spider_middleware.get_processed_request(+ request, response+ )++ def _engine_started(self) -> None:+ self._referer_spider_middleware = self.crawler.get_spider_middleware(RefererMiddleware)+ if self._referer_spider_middleware:+ return+ redirect_cls = global_object_name(self.__class__)+ referer_cls = global_object_name(RefererMiddleware)+ if self.__class__ in (RedirectMiddleware, MetaRefreshMiddleware):+ replacement = (+ f"replace {redirect_cls} with a subclass that overrides the "+ f"handle_referer() method"+ )+ else:+ replacement = (+ f"or edit {redirect_cls} (if defined in your code base) to "+ f"override the handle_referer() method, or replace "+ f"{redirect_cls} with a subclass that overrides the "+ f"handle_referer() method."+ )+ logger.warning(+ f"{redirect_cls} found no {referer_cls} instance to handle "+ f"Referer header handling, so the Referer header will be removed "+ f"on redirects. To set a Referer header on redirects, enable "+ f"{referer_cls} (or a subclass), or {replacement}.",+ )+def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request:ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)redirects = request.meta.get("redirect_times", 0) + 1@@ -132,17 +120,79 @@ class BaseRedirectMiddleware:)raise IgnoreRequest("max redirections reached")++ def _build_redirect_request(+ self, source_request: Request, response: Response, *, url: str, **kwargs: Any+ ) -> Request:+ redirect_request = source_request.replace(+ url=url,+ **kwargs,+ cls=None,+ cookies=None,+ )+ if "_scheme_proxy" in redirect_request.meta:+ source_request_scheme = urlparse_cached(source_request).scheme+ redirect_request_scheme = urlparse_cached(redirect_request).scheme+ if source_request_scheme != redirect_request_scheme:+ redirect_request.meta.pop("_scheme_proxy")+ redirect_request.meta.pop("proxy", None)+ redirect_request.meta.pop("_auth_proxy", None)+ redirect_request.headers.pop(b"Proxy-Authorization", None)++ has_cookie_header = "Cookie" in redirect_request.headers+ has_authorization_header = "Authorization" in redirect_request.headers+ if has_cookie_header or has_authorization_header:+ default_ports = {"http": 80, "https": 443}++ parsed_source_request = urlparse_cached(source_request)+ source_scheme, source_host, source_port = (+ parsed_source_request.scheme,+ parsed_source_request.hostname,+ parsed_source_request.port+ or default_ports.get(parsed_source_request.scheme),+ )++ parsed_redirect_request = urlparse_cached(redirect_request)+ redirect_scheme, redirect_host, redirect_port = (+ parsed_redirect_request.scheme,+ parsed_redirect_request.hostname,+ parsed_redirect_request.port+ or default_ports.get(parsed_redirect_request.scheme),+ )++ if has_cookie_header and (+ redirect_scheme not in {source_scheme, "https"}+ or source_host != redirect_host+ ):+ del redirect_request.headers["Cookie"]++ # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name+ if has_authorization_header and (+ source_scheme != redirect_scheme+ or source_host != redirect_host+ or source_port != redirect_port+ ):+ del redirect_request.headers["Authorization"]++ self.handle_referer(redirect_request, response)++ return redirect_request+def _redirect_request_using_get(- self, request: Request, redirect_url: str+ self, request: Request, response: Response, redirect_url: str) -> Request:- redirect_request = _build_redirect_request(+ redirect_request = self._build_redirect_request(request,+ response,url=redirect_url,method="GET",body="",)redirect_request.headers.pop("Content-Type", None)redirect_request.headers.pop("Content-Length", None)+ redirect_request.headers.pop("Content-Encoding", None)+ redirect_request.headers.pop("Content-Language", None)+ redirect_request.headers.pop("Content-Location", None)return redirect_request@@ -176,14 +226,22 @@ class RedirectMiddleware(BaseRedirectMiddleware):location = request_scheme + "://" + location.lstrip("/")redirected_url = urljoin(request.url, location)- redirected = _build_redirect_request(request, url=redirected_url)++ if not urlparse(redirected_url).fragment:+ fragment = urlparse_cached(request).fragment+ if fragment:+ redirected_url = urljoin(redirected_url, f"#{fragment}")++ redirected = self._build_redirect_request(request, response, url=redirected_url)if urlparse_cached(redirected).scheme not in {"http", "https"}:return response- if response.status in (307, 308) or request.method == "HEAD":- return self._redirect(redirected, request, response.status)+ if (+ (response.status in (301, 302) and request.method == "POST")+ or (response.status == 303 and request.method not in ("GET", "HEAD"))+ ):+ redirected = self._redirect_request_using_get(request, response, redirected_url)- redirected = self._redirect_request_using_get(request, redirected_url)return self._redirect(redirected, request, response.status)@@ -210,7 +268,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)if not url:return response- redirected = self._redirect_request_using_get(request, url)+ redirected = self._redirect_request_using_get(request, response, url)if urlparse_cached(redirected).scheme not in {"http", "https"}:return responseif cast("float", interval) < self._maxdelay:diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex e4d0f4014..5825f6467 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -9,6 +9,7 @@ import warningsfrom abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, castfrom urllib.parse import urlparse+from warnings import warnfrom w3lib.url import safe_url_string@@ -342,29 +343,47 @@ class RefererMiddleware(BaseSpiderMiddleware):def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("REFERER_ENABLED"):raise NotConfigured- mw = cls(crawler.settings)+ return cls(crawler.settings)- # Note: this hook is a bit of a hack to intercept redirections- crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)-- return mw-- def policy(self, resp_or_url: Response | str, request: Request) -> ReferrerPolicy:- """- Determine Referrer-Policy to use from a parent Response (or URL),- and a Request to be sent.+ def policy(+ self,+ response: Response | str | None = None,+ request: Request | None = None,+ **kwargs,+ ) -> ReferrerPolicy:+ """Return the referrer policy to use for *request* based on *request*+ meta, *response* and settings.- if a valid policy is set in Request meta, it is used.- - if the policy is set in meta but is wrong (e.g. a typo error),- the policy from settings is used- - if the policy is not set in Request meta,- but there is a Referrer-policy header in the parent response,- it is used if valid+ - if the policy is set in meta but is wrong (e.g. a typo error), the+ policy from settings is used+ - if the policy is not set in Request meta, but there is a+ Referrer-Policy header in the parent response, it is used if valid- otherwise, the policy from settings is used."""+ if "resp_or_url" in kwargs:+ if response is not None:+ raise TypeError("Cannot pass both 'response' and 'resp_or_url'")+ response = kwargs.pop("resp_or_url")+ warn(+ "Passing 'resp_or_url' is deprecated, use 'response' instead.",+ DeprecationWarning,+ stacklevel=2,+ )+ if response is None:+ raise TypeError("Missing required argument: 'response'")+ if request is None:+ raise TypeError("Missing required argument: 'request'")+ if isinstance(response, str):+ warn(+ "Passing a response URL to RefererMiddleware.policy() instead "+ "of a Response object is deprecated.",+ DeprecationWarning,+ stacklevel=2,+ )policy_name = request.meta.get("referrer_policy")- if policy_name is None and isinstance(resp_or_url, Response):- policy_header = resp_or_url.headers.get("Referrer-Policy")+ if policy_name is None and isinstance(response, Response):+ policy_header = response.headers.get("Referrer-Policy")if policy_header is not None:policy_name = to_unicode(policy_header.decode("latin1"))if policy_name is None:@@ -383,25 +402,3 @@ class RefererMiddleware(BaseSpiderMiddleware):if referrer is not None:request.headers.setdefault("Referer", referrer)return request-- def request_scheduled(self, request: Request, spider: Spider) -> None:- # check redirected request to patch "Referer" header if necessary- redirected_urls = request.meta.get("redirect_urls", [])- if redirected_urls:- request_referrer = request.headers.get("Referer")- # we don't patch the referrer value if there is none- if request_referrer is not None:- # the request's referrer header value acts as a surrogate- # for the parent response URL- #- # Note: if the 3xx response contained a Referrer-Policy header,- # the information is not available using this hook- parent_url = safe_url_string(request_referrer)- policy_referrer = self.policy(parent_url, request).referrer(- parent_url, request.url- )- if policy_referrer != request_referrer.decode("latin1"):- if policy_referrer is None:- request.headers.pop("Referer")- else:- request.headers["Referer"] = policy_referrerdiff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 421b654a7..4197c7c86 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -1,4 +1,6 @@+import loggingfrom itertools import chain, product+from unittest.mock import MagicMockimport pytest@@ -9,8 +11,14 @@ from scrapy.downloadermiddlewares.redirect import ()from scrapy.exceptions import IgnoreRequestfrom scrapy.http import HtmlResponse, Request, Response+from scrapy.spidermiddlewares.referer import (+ POLICY_NO_REFERRER,+ POLICY_ORIGIN,+ POLICY_UNSAFE_URL,+ RefererMiddleware,+)from scrapy.spiders import Spider-from scrapy.utils.misc import set_environ+from scrapy.utils.misc import build_from_crawler, set_environfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -18,8 +26,8 @@ from scrapy.utils.test import get_crawlerclass Base:class Test:def test_priority_adjust(self):- req = Request("http://a.com")- rsp = self.get_response(req, "http://a.com/redirected")+ req = Request("http://a.example")+ rsp = self.get_response(req, "http://a.example/redirected")req2 = self.mw.process_response(req, rsp)assert req2.priority > req.priority@@ -65,7 +73,7 @@ class Base:def test_max_redirect_times(self):self.mw.max_redirect_times = 1- req = Request("http://scrapytest.org/302")+ req = Request("http://a.example/302")rsp = self.get_response(req, "/redirected")req = self.mw.process_response(req, rsp)@@ -77,7 +85,7 @@ class Base:def test_ttl(self):self.mw.max_redirect_times = 100- req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1})+ req = Request("http://a.example/302", meta={"redirect_ttl": 1})rsp = self.get_response(req, "/a")req = self.mw.process_response(req, rsp)@@ -86,22 +94,22 @@ class Base:self.mw.process_response(req, rsp)def test_redirect_urls(self):- req1 = Request("http://scrapytest.org/first")+ req1 = Request("http://a.example/first")rsp1 = self.get_response(req1, "/redirected")req2 = self.mw.process_response(req1, rsp1)rsp2 = self.get_response(req1, "/redirected2")req3 = self.mw.process_response(req2, rsp2)- assert req2.url == "http://scrapytest.org/redirected"- assert req2.meta["redirect_urls"] == ["http://scrapytest.org/first"]- assert req3.url == "http://scrapytest.org/redirected2"+ assert req2.url == "http://a.example/redirected"+ assert req2.meta["redirect_urls"] == ["http://a.example/first"]+ assert req3.url == "http://a.example/redirected2"assert req3.meta["redirect_urls"] == [- "http://scrapytest.org/first",- "http://scrapytest.org/redirected",+ "http://a.example/first",+ "http://a.example/redirected",]def test_redirect_reasons(self):- req1 = Request("http://scrapytest.org/first")+ req1 = Request("http://a.example/first")rsp1 = self.get_response(req1, "/redirected1")req2 = self.mw.process_response(req1, rsp1)rsp2 = self.get_response(req2, "/redirected2")@@ -1046,8 +1054,158 @@ class TestRedirectMiddleware(Base.Test):assert "Content-Length" not in redirect_request.headersassert not redirect_request.body+ @pytest.mark.parametrize("status", [301, 302])+ @pytest.mark.parametrize("method", ["PUT", "DELETE"])+ def test_method_not_converted_on_301_302(self, status, method):+ url = f"http://www.example.com/{status}"+ url2 = "http://www.example.com/redirected"+ body = b"test-body"+ req = Request(+ url,+ method=method,+ body=body,+ headers={"Content-Type": "text/plain", "Content-Length": str(len(body))},+ )+ rsp = Response(url, headers={"Location": url2}, status=status)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == url2+ assert req2.method == method+ assert req2.body == body+ assert req2.headers[b"Content-Type"] == b"text/plain"+ assert req2.headers[b"Content-Length"] == str(len(body)).encode()++ @pytest.mark.parametrize("method", ["PUT", "DELETE"])+ def test_method_converted_on_303(self, method):+ status = 303+ url = f"http://www.example.com/{status}"+ url2 = "http://www.example.com/redirected"+ req = Request(url, method=method)+ rsp = Response(url, headers={"Location": url2}, status=status)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == url2+ assert req2.method == "GET"++ def test_get_method_body_preserved_on_303(self):+ status = 303+ url = f"http://www.example.com/{status}"+ url2 = "http://www.example.com/redirected"+ body = b"test-body"+ req = Request(+ url,+ method="GET",+ body=body,+ headers={"Content-Type": "text/plain", "Content-Length": str(len(body))},+ )+ rsp = Response(url, headers={"Location": url2}, status=status)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == url2+ assert req2.method == "GET"+ assert req2.body == body+ assert req2.headers[b"Content-Type"] == b"text/plain"+ assert req2.headers[b"Content-Length"] == str(len(body)).encode()++ def test_redirect_strips_content_headers(self):+ url = "http://www.example.com/303"+ url2 = "http://www.example.com/redirected"+ headers = {+ "Content-Type": "application/json",+ "Content-Length": "100",+ "Content-Encoding": "gzip",+ "Content-Language": "en",+ "Content-Location": "http://www.example.com/original",+ "X-Custom": "foo",+ }+ req = Request(url, method="POST", headers=headers, body=b"foo")+ rsp = Response(url, headers={"Location": url2}, status=303)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == url2+ assert req2.method == "GET"+ assert req2.body == b""+ assert "Content-Type" not in req2.headers+ assert "Content-Length" not in req2.headers+ assert "Content-Encoding" not in req2.headers+ assert "Content-Language" not in req2.headers+ assert "Content-Location" not in req2.headers+ assert req2.headers["X-Custom"] == b"foo"++ @pytest.mark.parametrize(+ (+ "referer",+ "source_url",+ "target_url",+ "policy_header",+ "expected_referer",+ ),+ [+ (+ None,+ "http://www.example.com/302",+ "http://www.example.com/redirected",+ None,+ b"http://www.example.com/302",+ ),+ (+ "http://example.com/old",+ "http://www.example.com/302",+ "http://www.example.com/redirected",+ None,+ b"http://www.example.com/302",+ ),+ (+ "https://example.com/old",+ "https://www.example.com/302",+ "http://www.example.com/redirected",+ None,+ None,+ ),+ (+ "http://example.com/old",+ "http://www.example.com/foo/bar",+ "http://www.example.com/redirected",+ "origin",+ b"http://www.example.com/",+ ),+ ],+ )+ def test_redirect_referer(+ self, referer, source_url, target_url, policy_header, expected_referer+ ):+ headers = {"Referer": referer} if referer else {}+ source_request = Request(source_url, headers=headers)+ resp_headers = {"Location": target_url}+ if policy_header:+ resp_headers["Referrer-Policy"] = policy_header+ response = Response(source_url, headers=resp_headers, status=302)+ crawler = get_crawler()+ referer_mw = build_from_crawler(RefererMiddleware, crawler)+ redirect_mw = self.mwcls.from_crawler(crawler)+ redirect_mw._referer_spider_middleware = referer_mw+ redirect_request = redirect_mw.process_response(source_request, response)+ if expected_referer:+ assert redirect_request.headers.get("Referer") == expected_referer+ else:+ assert "Referer" not in redirect_request.headers++ def test_redirect_strips_referer_no_middleware(self):+ source_url = "http://www.example.com/302"+ redirect_url = "http://www.example.com/redirected"+ source_request = Request(source_url, headers={"Referer": "http://example.com/old"})+ response = Response(source_url, headers={"Location": redirect_url}, status=302)+ redirect_mw = self.mwcls.from_crawler(get_crawler())+ redirect_mw._referer_spider_middleware = None+ redirect_request = redirect_mw.process_response(source_request, response)+ assert "Referer" not in redirect_request.headers+@pytest.mark.parametrize("status", [307, 308])- def test_cross_origin_strip_body(self, status):+ def test_cross_origin_maintain_body(self, status):source_url = "https://example.com"target_url = "https://attacker.example"body = b"secret"@@ -1062,9 +1220,19 @@ class TestRedirectMiddleware(Base.Test):assert isinstance(redirect_request, Request)assert redirect_request.url == target_urlassert redirect_request.method == "POST"- assert not getattr(redirect_request, "body", None)- assert b"Content-Type" not in redirect_request.headers- assert b"Content-Length" not in redirect_request.headers+ assert redirect_request.body == body+ assert redirect_request.headers[b"Content-Type"] == b"application/json"+ assert redirect_request.headers[b"Content-Length"] == str(len(body)).encode()++ def test_redirect_keeps_fragment(self):+ url = "http://www.example.com/301#frag"+ url2 = "http://www.example.com/redirected"+ req = Request(url)+ rsp = Response(url, headers={"Location": url2}, status=302)++ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.url == "http://www.example.com/redirected#frag"def test_redirect_302_head(self):url = "http://www.example.com/302"@@ -1311,4 +1479,114 @@ def test_meta_refresh_schemes(url, location, target):assert redirect == responseelse:assert isinstance(redirect, Request)- assert redirect.url == target++ ("policy", "source_url", "target_url", "expected_referrer"),+ [+ # The policy header affects the outcome.+ # (without it, the https → http switch would drop the referer)+ (+ POLICY_UNSAFE_URL,+ "https://a.example/1",+ "http://a.example/2",+ b"https://a.example/1",+ ),+ # The policy header can get the Referer header removed.+ (+ POLICY_NO_REFERRER,+ "http://a.example/1",+ "http://a.example/2",+ None,+ ),+ # The policy header can get the Referer header edited (path stripped).+ (+ POLICY_ORIGIN,+ "http://a.example/1",+ "http://a.example/2",+ b"http://a.example/",+ ),+ ],+)+def test_response_referrer_policy(+ policy, source_url, target_url, expected_referrer+):+ crawler = get_crawler()+ referrer_mw = build_from_crawler(RefererMiddleware, crawler)+ redirect_mw = build_from_crawler(RedirectMiddleware, crawler)+ redirect_mw._referer_spider_middleware = referrer_mw+ source_request = Request(source_url)+ extra_headers = {}+ if policy:+ extra_headers["Referrer-Policy"] = policy+ response_redirect = Response(+ source_request.url, status=301, headers={"Location": target_url, **extra_headers}+ )+ source_request = redirect_mw.process_response(+ source_request, response_redirect+ )+ assert isinstance(source_request, Request)++ assert source_request.headers.get("Referer") == expected_referrer+++def test_no_warning_when_referer_middleware_present(caplog):+ crawler = get_crawler()+ crawler.get_spider_middleware = MagicMock(return_value=MagicMock())+ mw = build_from_crawler(RedirectMiddleware, crawler)+ mw._engine_started()+ assert not caplog.records+++def test_warning_redirect_middleware(caplog):+ crawler = get_crawler()+ crawler.get_spider_middleware = MagicMock(return_value=None)+ mw = build_from_crawler(RedirectMiddleware, crawler)+ with caplog.at_level(logging.WARNING):+ mw._engine_started()+ assert (+ "scrapy.downloadermiddlewares.redirect.RedirectMiddleware found no "+ "scrapy.spidermiddlewares.referer.RefererMiddleware"+ ) in caplog.text+ assert "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" in caplog.text+ assert (+ "replace scrapy.downloadermiddlewares.redirect.RedirectMiddleware "+ "with a subclass that overrides the handle_referer() method"+ ) in caplog.text+++def test_warning_meta_refresh_middleware(caplog):+ crawler = get_crawler()+ crawler.get_spider_middleware = MagicMock(return_value=None)+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)+ with caplog.at_level(logging.WARNING):+ mw._engine_started()+ assert (+ "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no "+ "scrapy.spidermiddlewares.referer.RefererMiddleware"+ ) in caplog.text+ assert "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" in caplog.text+ assert (+ "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware "+ "with a subclass that overrides the handle_referer() method"+ ) in caplog.text+++def test_warning_subclass(caplog):+ class MyRedirectMiddleware(RedirectMiddleware):+ pass++ crawler = get_crawler()+ crawler.get_spider_middleware = MagicMock(return_value=None)+ mw = build_from_crawler(MyRedirectMiddleware, crawler)+ with caplog.at_level(logging.WARNING):+ mw._engine_started()+ assert (+ "test_warning_subclass.<locals>.MyRedirectMiddleware found no "+ "scrapy.spidermiddlewares.referer.RefererMiddleware"+ ) in caplog.text+ assert "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" in caplog.text+ assert "edit " in caplog.text+ assert "test_warning_subclass.<locals>.MyRedirectMiddleware" in caplog.text+ assert (+ "(if defined in your code base) to override the handle_referer() method"+ ) in caplog.textdiff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 68fcad98c..b4090b78c 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -6,7 +6,6 @@ from urllib.parse import urlparseimport pytest-from scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.spidermiddlewares.referer import (@@ -31,6 +30,7 @@ from scrapy.spidermiddlewares.referer import (StrictOriginWhenCrossOriginPolicy,UnsafeUrlPolicy,)+from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -970,391 +970,58 @@ class TestPolicyHeaderPrecedence004(resp_headers = {"Referrer-Policy": ""}-class TestReferrerOnRedirect(TestRefererMiddleware):- settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}- scenarii: Sequence[- tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None]- ] = [ # type: ignore[assignment]- (- "http://scrapytest.org/1", # parent- "http://scrapytest.org/2", # target- (- # redirections: code, URL- (301, "http://scrapytest.org/3"),- (301, "http://scrapytest.org/4"),- ),- b"http://scrapytest.org/1", # expected initial referer- b"http://scrapytest.org/1", # expected referer for the redirection request- ),- (- "https://scrapytest.org/1",- "https://scrapytest.org/2",- (- # redirecting to non-secure URL- (301, "http://scrapytest.org/3"),- ),- b"https://scrapytest.org/1",- b"https://scrapytest.org/1",- ),- (- "https://scrapytest.org/1",- "https://scrapytest.com/2",- (- # redirecting to non-secure URL: different origin- (301, "http://scrapytest.com/3"),- ),- b"https://scrapytest.org/1",- b"https://scrapytest.org/1",- ),- ]-- @pytest.fixture- def crawler(self) -> Crawler:- crawler = get_crawler(DefaultSpider, self.settings)- crawler.spider = crawler._create_spider()- return crawler-- @pytest.fixture- def referrermw(self, crawler: Crawler) -> RefererMiddleware:- return RefererMiddleware.from_crawler(crawler)-- @pytest.fixture- def redirectmw(self, crawler: Crawler) -> RedirectMiddleware:- return RedirectMiddleware.from_crawler(crawler)-- def test( # type: ignore[override]- self,- crawler: Crawler,- referrermw: RefererMiddleware,- redirectmw: RedirectMiddleware,- ) -> None:- for (- parent,- target,- redirections,- init_referrer,- final_referrer,- ) in self.scenarii:- response = self.get_response(parent)- request = self.get_request(target)-- out = list(referrermw.process_spider_output(response, [request]))- assert out[0].headers.get("Referer") == init_referrer-- for status, url in redirections:- response = Response(- request.url, headers={"Location": url}, status=status- )- request = cast(- "Request", redirectmw.process_response(request, response)- )- assert crawler.spider- referrermw.request_scheduled(request, crawler.spider)-- assert isinstance(request, Request)- assert request.headers.get("Referer") == final_referrer---class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):- """- No Referrer policy never sets the "Referer" header.- HTTP redirections should not change that.- """-- settings = {"REFERRER_POLICY": "no-referrer"}- scenarii = [- (- "http://scrapytest.org/1", # parent- "http://scrapytest.org/2", # target- (- # redirections: code, URL- (301, "http://scrapytest.org/3"),- (301, "http://scrapytest.org/4"),- ),- None, # expected initial "Referer"- None, # expected "Referer" for the redirection request- ),- (- "https://scrapytest.org/1",- "https://scrapytest.org/2",- ((301, "http://scrapytest.org/3"),),- None,- None,- ),- (- "https://scrapytest.org/1",- "https://example.com/2", # different origin- ((301, "http://scrapytest.com/3"),),- None,- None,- ),- ]---class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):- """- Same Origin policy sends the full URL as "Referer" if the target origin- is the same as the parent response (same protocol, same domain, same port).-- HTTP redirections to a different domain or a lower secure level- should have the "Referer" removed.- """-- settings = {"REFERRER_POLICY": "same-origin"}- scenarii = [- (- "http://scrapytest.org/101", # origin- "http://scrapytest.org/102", # target- (- # redirections: code, URL- (301, "http://scrapytest.org/103"),- (301, "http://scrapytest.org/104"),- ),- b"http://scrapytest.org/101", # expected initial "Referer"- b"http://scrapytest.org/101", # expected referer for the redirection request- ),- (- "https://scrapytest.org/201",- "https://scrapytest.org/202",- (- # redirecting from secure to non-secure URL == different origin- (301, "http://scrapytest.org/203"),- ),- b"https://scrapytest.org/201",- None,- ),- (- "https://scrapytest.org/301",- "https://scrapytest.org/302",- (- # different domain == different origin- (301, "http://example.com/303"),- ),- b"https://scrapytest.org/301",- None,- ),- ]-+class TestPolicyMethodResponseParamRename:+ def setup_method(self):+ self.crawler = get_crawler()+ self.mw = build_from_crawler(RefererMiddleware, self.crawler)+ self.request = Request("http://www.example.com")+ self.response = Response("http://www.example.com")++ def test_pos_string(self):+ with warnings.catch_warnings(record=True) as w:+ self.mw.policy("http://old.com", self.request)+ found = False+ for warning in w:+ if "Passing a response URL" in str(warning.message):+ found = True+ break+ assert found++ def test_pos_response(self):+ with warnings.catch_warnings(record=True) as w:+ self.mw.policy(self.response, self.request)+ for warning in w:+ assert "resp_or_url" not in str(warning.message)++ def test_key_resp_or_url(self):+ with warnings.catch_warnings(record=True) as w:+ self.mw.policy(resp_or_url=self.response, request=self.request)+ found = False+ for warning in w:+ if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str(warning.message):+ found = True+ break+ assert found++ def test_key_response(self):+ with warnings.catch_warnings(record=True) as w:+ self.mw.policy(response=self.response, request=self.request)+ for warning in w:+ assert "resp_or_url" not in str(warning.message)++ def test_key_response_string(self):+ with warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ self.mw.policy(response="http://old.com", request=self.request)+ found = False+ for warning in w:+ if "Passing a response URL" in str(warning.message):+ found = True+ break+ assert found++ def test_both_resp_or_url_and_response(self):+ with pytest.raises(TypeError, match="Cannot pass both 'response' and 'resp_or_url'"):+ self.mw.policy(response=self.response, resp_or_url=self.response, request=self.request)-class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):- """- Strict Origin policy will always send the "origin" as referrer- (think of it as the parent URL without the path part),- unless the security level is lower and no "Referer" is sent.-- Redirections from secure to non-secure URLs should have the- "Referrer" header removed if necessary.- """- settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN}- scenarii = [- (- "http://scrapytest.org/101",- "http://scrapytest.org/102",- (- (301, "http://scrapytest.org/103"),- (301, "http://scrapytest.org/104"),- ),- b"http://scrapytest.org/", # send origin- b"http://scrapytest.org/", # redirects to same origin: send origin- ),- (- "https://scrapytest.org/201",- "https://scrapytest.org/202",- (- # redirecting to non-secure URL: no referrer- (301, "http://scrapytest.org/203"),- ),- b"https://scrapytest.org/",- None,- ),- (- "https://scrapytest.org/301",- "https://scrapytest.org/302",- (- # redirecting to non-secure URL (different domain): no referrer- (301, "http://example.com/303"),- ),- b"https://scrapytest.org/",- None,- ),- (- "http://scrapy.org/401",- "http://example.com/402",- ((301, "http://scrapytest.org/403"),),- b"http://scrapy.org/",- b"http://scrapy.org/",- ),- (- "https://scrapy.org/501",- "https://example.com/502",- (- # HTTPS all along, so origin referrer is kept as-is- (301, "https://google.com/503"),- (301, "https://facebook.com/504"),- ),- b"https://scrapy.org/",- b"https://scrapy.org/",- ),- (- "https://scrapytest.org/601",- "http://scrapytest.org/602", # TLS to non-TLS: no referrer- (- (- 301,- "https://scrapytest.org/603",- ), # TLS URL again: (still) no referrer- ),- None,- None,- ),- ]---class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):- """- Origin When Cross-Origin policy sends the full URL as "Referer",- unless the target's origin is different (different domain, different protocol)- in which case only the origin is sent.-- Redirections to a different origin should strip the "Referer"- to the parent origin.- """-- settings = {"REFERRER_POLICY": POLICY_ORIGIN_WHEN_CROSS_ORIGIN}- scenarii = [- (- "http://scrapytest.org/101", # origin- "http://scrapytest.org/102", # target + redirection- (- # redirections: code, URL- (301, "http://scrapytest.org/103"),- (301, "http://scrapytest.org/104"),- ),- b"http://scrapytest.org/101", # expected initial referer- b"http://scrapytest.org/101", # expected referer for the redirection request- ),- (- "https://scrapytest.org/201",- "https://scrapytest.org/202",- (- # redirecting to non-secure URL: send origin- (301, "http://scrapytest.org/203"),- ),- b"https://scrapytest.org/201",- b"https://scrapytest.org/",- ),- (- "https://scrapytest.org/301",- "https://scrapytest.org/302",- (- # redirecting to non-secure URL (different domain): send origin- (301, "http://example.com/303"),- ),- b"https://scrapytest.org/301",- b"https://scrapytest.org/",- ),- (- "http://scrapy.org/401",- "http://example.com/402",- ((301, "http://scrapytest.org/403"),),- b"http://scrapy.org/",- b"http://scrapy.org/",- ),- (- "https://scrapy.org/501",- "https://example.com/502",- (- # all different domains: send origin- (301, "https://google.com/503"),- (301, "https://facebook.com/504"),- ),- b"https://scrapy.org/",- b"https://scrapy.org/",- ),- (- "https://scrapytest.org/301",- "http://scrapytest.org/302", # TLS to non-TLS: send origin- ((301, "https://scrapytest.org/303"),), # TLS URL again: send origin (also)- b"https://scrapytest.org/",- b"https://scrapytest.org/",- ),- ]---class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):- """- Strict Origin When Cross-Origin policy sends the full URL as "Referer",- unless the target's origin is different (different domain, different protocol)- in which case only the origin is sent...- Unless there's also a downgrade in security and then the "Referer" header- is not sent.-- Redirections to a different origin should strip the "Referer" to the parent origin,- and from https:// to http:// will remove the "Referer" header.- """-- settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}- scenarii = [- (- "http://scrapytest.org/101", # origin- "http://scrapytest.org/102", # target + redirection- (- # redirections: code, URL- (301, "http://scrapytest.org/103"),- (301, "http://scrapytest.org/104"),- ),- b"http://scrapytest.org/101", # expected initial referer- b"http://scrapytest.org/101", # expected referer for the redirection request- ),- (- "https://scrapytest.org/201",- "https://scrapytest.org/202",- (- # redirecting to non-secure URL: do not send the "Referer" header- (301, "http://scrapytest.org/203"),- ),- b"https://scrapytest.org/201",- None,- ),- (- "https://scrapytest.org/301",- "https://scrapytest.org/302",- (- # redirecting to non-secure URL (different domain): send origin- (301, "http://example.com/303"),- ),- b"https://scrapytest.org/301",- None,- ),- (- "http://scrapy.org/401",- "http://example.com/402",- ((301, "http://scrapytest.org/403"),),- b"http://scrapy.org/",- b"http://scrapy.org/",- ),- (- "https://scrapy.org/501",- "https://example.com/502",- (- # all different domains: send origin- (301, "https://google.com/503"),- (301, "https://facebook.com/504"),- ),- b"https://scrapy.org/",- b"https://scrapy.org/",- ),- (- "https://scrapytest.org/601",- "http://scrapytest.org/602", # TLS to non-TLS: do not send "Referer"- (- (- 301,- "https://scrapytest.org/603",- ), # TLS URL again: (still) send nothing- ),- None,- None,- ),- ]