Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add new "strict-" policies

Paul Tremberthc808a97c7471
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 24e5eac40..5d2a0e7b1 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -17,7 +17,9 @@ POLICY_NO_REFERRER = "no-referrer"
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
POLICY_SAME_ORIGIN = "same-origin"
POLICY_ORIGIN = "origin"
+POLICY_STRICT_ORIGIN = "strict-origin"
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
+POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"
@@ -139,6 +141,26 @@ class OriginPolicy(ReferrerPolicy):
return self.origin_referrer(response)
+class StrictOriginPolicy(ReferrerPolicy):
+ """
+ https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
+
+ The "strict-origin" policy sends the ASCII serialization
+ of the origin of the request client when making requests:
+ - from a TLS-protected environment settings object to a potentially trustworthy URL, and
+ - from non-TLS-protected environment settings objects to any origin.
+
+ Requests from TLS-protected request clients to non- potentially trustworthy URLs,
+ on the other hand, will contain no referrer information.
+ A Referer HTTP header will not be sent.
+ """
+ name = POLICY_STRICT_ORIGIN
+
+ def referrer(self, response, request):
+ if urlparse_cached(response).scheme == urlparse_cached(request).scheme:
+ return self.origin_referrer(response)
+
+
class OriginWhenCrossOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
@@ -160,6 +182,33 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
return origin
+class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
+ """
+ https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
+
+ The "strict-origin-when-cross-origin" policy specifies that a full URL,
+ stripped for use as a referrer, is sent as referrer information
+ when making same-origin requests from a particular request client,
+ and only the ASCII serialization of the origin of the request client
+ when making cross-origin requests:
+
+ - from a TLS-protected environment settings object to a potentially trustworthy URL, and
+ - from non-TLS-protected environment settings objects to any origin.
+
+ Requests from TLS-protected clients to non- potentially trustworthy URLs,
+ on the other hand, will contain no referrer information.
+ A Referer HTTP header will not be sent.
+ """
+ name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+
+ def referrer(self, response, request):
+ origin = self.origin(response)
+ if origin == self.origin(request):
+ return self.stripped_referrer(response)
+ else:
+ return origin
+
+
class UnsafeUrlPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url