Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use urlparse_cached() for OriginPolicy

Paul Tremberthf2ee6be3bb31
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index cbbcd3e97..bf2a3c037 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy):
name = POLICY_ORIGIN
def referrer(self, response, request):
- return self.strip_url(response.url, origin_only=True)
+ stripped = self.strip_url_parsed(response, origin_only=True)
+ if stripped is not None:
+ return urlunparse(stripped)
class OriginWhenCrossOriginPolicy(ReferrerPolicy):