Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix implementation
Paul Tremberth77aec5a79681
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 5d2a0e7b1..a6316cd0c 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -31,13 +31,13 @@ class ReferrerPolicy(object):def referrer(self, response, request):raise NotImplementedError()- def stripped_referrer(self, req_or_resp):- return self.strip_url(req_or_resp)+ def stripped_referrer(self, r):+ return self.strip_url(r)- def origin_referrer(self, req_or_resp):- return self.strip_url(req_or_resp, origin_only=True)+ def origin_referrer(self, r):+ return self.strip_url(r, origin_only=True)- def strip_url(self, req_or_resp, origin_only=False):+ def strip_url(self, r, origin_only=False):"""https://www.w3.org/TR/referrer-policy/#strip-url@@ -51,9 +51,9 @@ class ReferrerPolicy(object):Set url's query to null.Return url."""- if req_or_resp.url is None or not req_or_resp.url:+ if r is None or not r.url:return None- parsed_url = urlparse_cached(req_or_resp)+ parsed_url = urlparse_cached(r)if parsed_url.scheme not in self.NOREFERRER_SCHEMES:return strip_url(parsed_url,strip_credentials=True,@@ -61,9 +61,19 @@ class ReferrerPolicy(object):strip_default_port=True,origin_only=origin_only)- def origin(self, req_or_resp):+ def origin(self, r):"""Return serialized origin (scheme, host, path) for a request or response URL."""- return self.strip_url(req_or_resp, origin_only=True)+ return self.strip_url(r, origin_only=True)++ def potentially_trustworthy(self, r):+ # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy+ parsed_url = urlparse_cached(r)+ if parsed_url.scheme in ('data',):+ return False+ return self.tls_protected(r)++ def tls_protected(self, r):+ return urlparse_cached(r).scheme in ('https', 'ftps')class NoReferrerPolicy(ReferrerPolicy):@@ -157,7 +167,9 @@ class StrictOriginPolicy(ReferrerPolicy):name = POLICY_STRICT_ORIGINdef referrer(self, response, request):- if urlparse_cached(response).scheme == urlparse_cached(request).scheme:+ if ((urlparse_cached(response).scheme == 'https' and+ self.potentially_trustworthy(request))+ or urlparse_cached(response).scheme == 'http'):return self.origin_referrer(response)@@ -205,8 +217,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):origin = self.origin(response)if origin == self.origin(request):return self.stripped_referrer(response)- else:- return origin+ elif ((urlparse_cached(response).scheme in ('https', 'ftps') and+ self.potentially_trustworthy(request))+ or urlparse_cached(response).scheme == 'http'):+ return self.origin_referrer(response)class UnsafeUrlPolicy(ReferrerPolicy):@@ -244,7 +258,9 @@ _policy_classes = {p.name: p for p in (NoReferrerWhenDowngradePolicy,SameOriginPolicy,OriginPolicy,+ StrictOriginPolicy,OriginWhenCrossOriginPolicy,+ StrictOriginWhenCrossOriginPolicy,UnsafeUrlPolicy,DefaultReferrerPolicy,)}