Scrapy, a fast high-level web crawling & scraping framework for Python.
WIP Add Referrer policies
Paul Tremberthbaed7c436f8c
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 6a8c46543..21b340f22 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -2,22 +2,226 @@RefererMiddleware: populates Request referer field, based on the Response whichoriginated it."""+from six.moves.urllib.parse import urlsplit, urlunsplitfrom scrapy.http import Requestfrom scrapy.exceptions import NotConfigured+from scrapy.utils.python import to_native_str+++LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)++class ReferrerPolicy(object):++ NOREFERRER_SCHEMES = LOCAL_SCHEMES++ def referrer(self, response, request):+ raise NotImplementedError()++ def strip_url(self, url, origin_only=False):+ """+ https://www.w3.org/TR/referrer-policy/#strip-url++ If url is null, return no referrer.+ If url's scheme is a local scheme, then return no referrer.+ Set url's username to the empty string.+ Set url's password to null.+ Set url's fragment to null.+ If the origin-only flag is true, then:+ Set url's path to null.+ Set url's query to null.+ Return url.+ """+ if url is None or not url:+ return None+ parsed = urlsplit(url, allow_fragments=True)++ if parsed.scheme in self.NOREFERRER_SCHEMES:+ return None+ if parsed.username or parsed.password:+ netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')+ else:+ netloc = parsed.netloc+ return urlunsplit((+ parsed.scheme,+ netloc,+ '' if origin_only else parsed.path,+ '' if origin_only else parsed.query,+ ''))+++class NoReferrerPolicy(ReferrerPolicy):+ """+ https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer++ The simplest policy is "no-referrer", which specifies that no referrer information+ is to be sent along with requests made from a particular request client to any origin.+ The header will be omitted entirely.+ """+ name = "no-referrer"++ def referrer(self, response, request):+ return None+++class NoReferrerWhenDowngradePolicy(ReferrerPolicy):+ """+ https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade++ The "no-referrer-when-downgrade" policy sends a full URL+ along with requests from a TLS-protected environment settings object+ to a a priori authenticated URL,+ and requests from request clients which are not TLS-protected to any origin.++ Requests from TLS-protected request clients to non-a priori authenticated URLs,+ on the other hand, will contain no referrer information.+ A Referer HTTP header will not be sent.++ This is a user agent's default behavior, if no policy is otherwise specified.+ """+ name = "no-referrer-when-downgrade"++ def referrer(self, response, request):+ target_url = request.url++ referrer_source = response.url+ referrer_url = self.strip_url(referrer_source)++ # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer:+ #+ # If environment is TLS-protected+ # and the origin of request's current URL is not an a priori authenticated URL,+ # then return no referrer.+ if urlsplit(referrer_source).scheme in ('https', 'ftps') and \+ urlsplit(target_url).scheme in ('http',):+ return None+ return referrer_url+++class SameOriginPolicy(ReferrerPolicy):+ """+ https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin++ The "same-origin" policy specifies that a full URL, stripped for use as a referrer,+ is sent as referrer information when making same-origin requests from a particular request client.++ Cross-origin requests, on the other hand, will contain no referrer information.+ A Referer HTTP header will not be sent.+ """+ name = "same-origin"++ def referrer(self, response, request):+ target_url = request.url+ referrer_source = response.url+ if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc:+ return self.strip_url(referrer_source)+ else:+ return None+++class OriginPolicy(ReferrerPolicy):+ """+ https://www.w3.org/TR/referrer-policy/#referrer-policy-origin++ The "origin" policy specifies that only the ASCII serialization+ of the origin of the request client is sent as referrer information+ when making both same-origin requests and cross-origin requests+ from a particular request client.+ """+ name = "origin"++ def referrer(self, response, request):+ return self.strip_url(referrer_source, origin_only=True)+++class OriginWhenCrossOriginPolicy(ReferrerPolicy):+ """+ https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin++ The "origin-when-cross-origin" policy specifies that a full URL,+ stripped for use as a referrer, is sent as referrer information+ when making same-origin requests from a particular request client,+ and only the ASCII serialization of the origin of the request client+ is sent as referrer information when making cross-origin requests+ from a particular request client.+ """+ name = "origin-when-cross-origin"++ def referrer(self, response, request):+ target_url = request.url+ referrer_source = response.url++ # same origin --> send full referrer+ # different origin --> send only "origin" as referrer+ if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc:+ origin_only = True+ return self.strip_url(referrer_source, origin_only=origin_only)+++class UnsafeUrlPolicy(ReferrerPolicy):+ """+ https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url++ The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,+ is sent along with both cross-origin requests+ and same-origin requests made from a particular request client.++ Note: The policy's name doesn't lie; it is unsafe.+ This policy will leak origins and paths from TLS-protected resources+ to insecure origins.+ Carefully consider the impact of setting such a policy for potentially sensitive documents.+ """+ name = "unsafe-url"++ def referrer(self, response, request):+ referrer_source = response.url+ return self.strip_url(referrer_source)+++class LegacyPolicy(ReferrerPolicy):+ def referrer(self, response, request):+ return response.url+++class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):++ NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')+++_policies = {p.name: p for p in (+ NoReferrerPolicy,+ NoReferrerWhenDowngradePolicy,+ SameOriginPolicy,+ OriginPolicy,+ OriginWhenCrossOriginPolicy,+ UnsafeUrlPolicy,+)}class RefererMiddleware(object):+ def __init__(self, policy_class=DefaultReferrerPolicy):+ self.default_policy = policy_class+@classmethoddef from_crawler(cls, crawler):if not crawler.settings.getbool('REFERER_ENABLED'):raise NotConfiguredreturn cls()+ def policy(self, response, request):+ policy_name = request.meta.get('referrer_policy')+ if policy_name is None:+ policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1'))++ policy_class = _policies.get(policy_name.lower(), self.default_policy)+ return policy_class()+def process_spider_output(self, response, result, spider):def _set_referer(r):if isinstance(r, Request):- r.headers.setdefault('Referer', response.url)+ referrer = self.policy(response, r).referrer(response, r)+ if referrer is not None:+ r.headers.setdefault('Referer', referrer)return rreturn (_set_referer(r) for r in result or ())diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex bd7673efb..f109bb248 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -19,3 +19,77 @@ class TestRefererMiddleware(TestCase):self.assertEquals(out[0].headers.get('Referer'),b'http://scrapytest.org')+ def test_policy_default(self):+ """+ Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade++ with some additional filtering of s3://+ """+ # a) https:// --> https:// -- include Referer header+ origin = Response('https://example.com/')+ target = Request('https://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'),+ b'https://example.com/')++ # b.1) http:// --> http:// -- include Referer header+ origin = Response('http://example.com/')+ target = Request('http://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'),+ b'http://example.com/')++ # b.2) http:// --> https:// -- include Referer header+ origin = Response('http://example.com/')+ target = Request('https://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'),+ b'http://example.com/')++ # c) https:// --> http:// -- Referer header NOT sent+ origin = Response('https://example.com/')+ target = Request('http://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'), None)++ def test_policy_default_no_credentials_leak(self):+ origin = Response('http://user:[email protected]/')+ target = Request('https://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'),+ b'http://example.com/')++ def test_policy_default_file_no_referrer_leak(self):+ # file:// --> https:// -- Referrer NOT sent+ origin = Response('file:///home/path/to/somefile.html')+ target = Request('https://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'), None)++ # file:// --> http:// -- Referrer NOT sent+ origin = Response('file:///home/path/to/somefile.html')+ target = Request('http://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'), None)++ def test_policy_default_s3_no_referrer_leak(self):+ # s3:// --> https:// -- Referrer NOT sent+ origin = Response('s3://mybucket/path/to/data.csv')+ target = Request('https://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'), None)++ # s3:// --> http:// -- Referrer NOT sent+ origin = Response('s3://mybucket/path/to/data.csv')+ target = Request('http://scrapy.org/')++ out = list(self.mw.process_spider_output(origin, [target], self.spider))+ self.assertEquals(out[0].headers.get('Referer'), None)