Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactor ReferrerPolicy methods
Paul Tremberthf6205778f31d
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 46351576f..44a599433 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -28,6 +28,16 @@ class ReferrerPolicy(object):def referrer(self, response, request):raise NotImplementedError()+ def stripped_referrer(self, req_or_resp):+ stripped = self.strip_url(req_or_resp)+ if stripped is not None:+ return urlunparse(stripped)++ def origin_referrer(self, req_or_resp):+ stripped = self.strip_url(req_or_resp, origin_only=True)+ if stripped is not None:+ return urlunparse(stripped)+def strip_url(self, req_or_resp, origin_only=False):"""https://www.w3.org/TR/referrer-policy/#strip-url@@ -113,9 +123,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):if urlparse_cached(response).scheme in ('https', 'ftps') and \urlparse_cached(request).scheme in ('http',):return None- stripped = self.strip_url(response)- if stripped is not None:- return urlunparse(stripped)+ return self.stripped_referrer(response)class SameOriginPolicy(ReferrerPolicy):@@ -132,9 +140,7 @@ class SameOriginPolicy(ReferrerPolicy):def referrer(self, response, request):if self.origin(response) == self.origin(request):- stripped = self.strip_url(response)- if stripped is not None:- return urlunparse(stripped)+ return self.stripped_referrer(response)class OriginPolicy(ReferrerPolicy):@@ -149,9 +155,7 @@ class OriginPolicy(ReferrerPolicy):name = POLICY_ORIGINdef referrer(self, response, request):- origin = self.strip_url(response, origin_only=True)- if origin is not None:- return urlunparse(origin)+ return self.origin_referrer(response)class OriginWhenCrossOriginPolicy(ReferrerPolicy):@@ -170,9 +174,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):def referrer(self, response, request):origin = self.origin(response)if origin == self.origin(request):- stripped = self.strip_url(response)- if stripped is not None:- return urlunparse(stripped)+ return self.stripped_referrer(response)else:return urlunparse(origin + ('', '', ''))@@ -193,9 +195,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):name = POLICY_UNSAFE_URLdef referrer(self, response, request):- stripped = self.strip_url(response)- if stripped is not None:- return urlunparse(stripped)+ return self.stripped_referrer(response)class LegacyPolicy(ReferrerPolicy):