Scrapy, a fast high-level web crawling & scraping framework for Python.
Use urlparse_cached() for OriginPolicy
Paul Tremberthf2ee6be3bb31
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex cbbcd3e97..bf2a3c037 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy):name = POLICY_ORIGINdef referrer(self, response, request):- return self.strip_url(response.url, origin_only=True)+ stripped = self.strip_url_parsed(response, origin_only=True)+ if stripped is not None:+ return urlunparse(stripped)class OriginWhenCrossOriginPolicy(ReferrerPolicy):