Scrapy, a fast high-level web crawling & scraping framework for Python.
Add tests for referrer policy via settings and via Request meta
Paul Tremberthe72b6e33611f
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 8458fe90b..b724d7999 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -1,233 +1,316 @@from unittest import TestCase+from scrapy.exceptions import NotConfiguredfrom scrapy.http import Response, Request+from scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.spidermiddlewares.referer import RefererMiddleware, \POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \- POLICY_UNSAFE_URL+ POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \+ DefaultReferrerPolicy, \+ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \+ OriginWhenCrossOriginPolicy, OriginPolicy, \+ SameOriginPolicy, UnsafeUrlPolicyclass TestRefererMiddleware(TestCase):+ req_meta = {}+ resp_headers = {}+ settings = {}+ scenarii = [+ ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),+ ]+def setUp(self):self.spider = Spider('foo')- self.mw = RefererMiddleware()-- def test_process_spider_output(self):- res = Response('http://scrapytest.org')- reqs = [Request('http://scrapytest.org/')]-- out = list(self.mw.process_spider_output(res, reqs, self.spider))- self.assertEquals(out[0].headers.get('Referer'),- b'http://scrapytest.org')-- def test_policy_default(self):- """- Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade-- with some additional filtering of s3://- """- for origin, target, referrer in [- ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),- ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),- ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),- ('https://example.com/', 'http://scrapy.org/', None),-- # no credentials leak- ('http://user:[email protected]/', 'https://scrapy.org/', b'http://example.com/'),-- # no referrer leak for local schemes- ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),- ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),-- # no referrer leak for s3 origins- ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),- ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),- ]:- response = Response(origin)- request = Request(target)+ settings = Settings(self.settings)+ self.mw = RefererMiddleware(settings)- out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEquals(out[0].headers.get('Referer'), referrer)+ def get_request(self, target):+ return Request(target, meta=self.req_meta)- def test_policy_no_referrer(self):+ def get_response(self, origin):+ return Response(origin, headers=self.resp_headers)- for origin, target, referrer in [- ('https://example.com/page.html', 'https://example.com/', None),- ('http://www.example.com/', 'https://scrapy.org/', None),- ('http://www.example.com/', 'http://scrapy.org/', None),- ('https://www.example.com/', 'http://scrapy.org/', None),- ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),- ]:- response = Response(origin)- request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER})+ def test(self):++ for origin, target, referrer in self.scenarii:+ response = self.get_response(origin)+ request = self.get_request(target)out = list(self.mw.process_spider_output(response, [request], self.spider))self.assertEquals(out[0].headers.get('Referer'), referrer)- def test_policy_no_referrer_when_downgrade(self):-- for origin, target, referrer in [- # TLS to TLS: send non-empty referrer- ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),- ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),- ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),- ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),- ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),-- # TLS to non-TLS: do not send referrer- ('https://example.com/page.html', 'http://not.example.com/', None),- ('https://example.com/page.html', 'http://scrapy.org/', None),- ('ftps://example.com/urls.zip', 'http://scrapy.org/', None),-- # non-TLS to TLS or non-TLS: send referrer- ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),- ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),- ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),- ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),- ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),- ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),- ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),- ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),-- # test for user/password stripping- ('http://user:[email protected]/page.html', 'https://not.example.com/', b'http://example.com/page.html'),- ]:- response = Response(origin)- request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE})- out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEquals(out[0].headers.get('Referer'), referrer)+class MixinDefault(object):+ """+ Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade++ with some additional filtering of s3://+ """+ scenarii = [+ ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),+ ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),+ ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),+ ('https://example.com/', 'http://scrapy.org/', None),++ # no credentials leak+ ('http://user:[email protected]/', 'https://scrapy.org/', b'http://example.com/'),++ # no referrer leak for local schemes+ ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),+ ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),++ # no referrer leak for s3 origins+ ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),+ ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),+ ]+++class MixinNoReferrer(object):+ scenarii = [+ ('https://example.com/page.html', 'https://example.com/', None),+ ('http://www.example.com/', 'https://scrapy.org/', None),+ ('http://www.example.com/', 'http://scrapy.org/', None),+ ('https://www.example.com/', 'http://scrapy.org/', None),+ ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),+ ]+++class MixinNoReferrerWhenDowngrade(object):+ scenarii = [+ # TLS to TLS: send non-empty referrer+ ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),+ ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),+ ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),+ ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),+ ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),++ # TLS to non-TLS: do not send referrer+ ('https://example.com/page.html', 'http://not.example.com/', None),+ ('https://example.com/page.html', 'http://scrapy.org/', None),+ ('ftps://example.com/urls.zip', 'http://scrapy.org/', None),++ # non-TLS to TLS or non-TLS: send referrer+ ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),+ ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),+ ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),+ ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),+ ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),+ ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),+ ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),+ ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),++ # test for user/password stripping+ ('http://user:[email protected]/page.html', 'https://not.example.com/', b'http://example.com/page.html'),+ ]+++class MixinSameOrigin(object):+ scenarii = [+ # Same origin (protocol, host, port): send referrer+ ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),+ ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),+ ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),+ ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),++ # Different host: do NOT send referrer+ ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),+ ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),+ ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),++ # Different port: do NOT send referrer+ ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),+ ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),+ ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),++ # Different protocols: do NOT send refferer+ ('https://example.com/page.html', 'http://example.com/not-page.html', None),+ ('https://example.com/page.html', 'http://not.example.com/', None),+ ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),+ ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),+ ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),++ # test for user/password stripping+ ('https://user:[email protected]/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('https://user:[email protected]/page.html', 'http://example.com/not-page.html', None),+ ]+++class MixinOrigin(object):+ scenarii = [+ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)+ ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),+ ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),+ ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),+ ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),++ # test for user/password stripping+ ('https://user:[email protected]/page.html', 'http://scrapy.org', b'https://example.com/'),+ ]+++class MixinOriginWhenCrossOrigin(object):+ scenarii = [+ # Same origin (protocol, host, port): send referrer+ ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),+ ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),+ ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),+ ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),++ # Different host: send origin as referrer+ ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),+ ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),+ ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),+ # exact match required+ ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),- def test_policy_same_origin(self):-- for origin, target, referrer in [- # Same origin (protocol, host, port): send referrer- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),- ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),- ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),- ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),- ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),- ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),-- # Different host: do NOT send referrer- ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),- ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),- ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),-- # Different port: do NOT send referrer- ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),- ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),- ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),-- # Different protocols: do NOT send refferer- ('https://example.com/page.html', 'http://example.com/not-page.html', None),- ('https://example.com/page.html', 'http://not.example.com/', None),- ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),- ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),- ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),-- # test for user/password stripping- ('https://user:[email protected]/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),- ('https://user:[email protected]/page.html', 'http://example.com/not-page.html', None),- ]:- response = Response(origin)- request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN})+ # Different port: send origin as referrer+ ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),+ ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),- out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEquals(out[0].headers.get('Referer'), referrer)+ # Different protocols: send origin as referrer+ ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),+ ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),+ ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),- def test_policy_origin(self):+ # test for user/password stripping+ ('https://user:[email protected]/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),+ # TLS to non-TLS downgrade: send origin+ ('https://user:[email protected]/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),+ ]- for origin, target, referrer in [- # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),- ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),- ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),- ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),- # test for user/password stripping- ('https://user:[email protected]/page.html', 'http://scrapy.org', b'https://example.com/'),- ]:- response = Response(origin)- request = Request(target, meta={'referrer_policy': POLICY_ORIGIN})+class MixinUnsafeUrl(object):+ scenarii = [+ # TLS to TLS: send referrer+ ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),+ ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),+ ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),+ ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),+ ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),+ ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),- out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEquals(out[0].headers.get('Referer'), referrer)+ # TLS to non-TLS: send referrer (yes, it's unsafe)+ ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),+ ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),+ ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),- def test_policy_origin_when_cross_origin(self):-- for origin, target, referrer in [- # Same origin (protocol, host, port): send referrer- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),- ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),- ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),- ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),- ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),- ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),-- # Different host: send origin as referrer- ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),- ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),- ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),- # exact match required- ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),-- # Different port: send origin as referrer- ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),- ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),-- # Different protocols: send origin as referrer- ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),- ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),- ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),-- # test for user/password stripping- ('https://user:[email protected]/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),- # TLS to non-TLS downgrade: send origin- ('https://user:[email protected]/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),- ]:- response = Response(origin)- request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN})+ # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)+ ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),+ ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),+ ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),+ ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),+ ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),+ ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),+ ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),+ ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),- out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEquals(out[0].headers.get('Referer'), referrer)+ # test for user/password stripping+ ('http://user:[email protected]/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),+ ('https://user:[email protected]/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),+ ]- def test_policy_unsafe_url(self):-- for origin, target, referrer in [- # TLS to TLS: send referrer- ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),- ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),- ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),- ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),- ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),- ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),-- # TLS to non-TLS: send referrer (yes, it's unsafe)- ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),- ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),- ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),-- # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)- ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),- ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),- ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),- ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),- ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),- ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),- ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),- ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),-- # test for user/password stripping- ('http://user:[email protected]/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),- ('https://user:[email protected]/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),- ]:- response = Response(origin)- request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL})- out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEquals(out[0].headers.get('Referer'), referrer)+class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):+ pass+++# --- Tests using settings to set policy using class path+class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}+++class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}+++class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}+++class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}+++class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}+++class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}+++# --- Tests using Request meta dict to set policy+class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}+++class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_NO_REFERRER}+++class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}+++class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}+++class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_ORIGIN}+++class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}+++class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_UNSAFE_URL}++++class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}+ req_meta = {'referrer_policy': POLICY_UNSAFE_URL}+++class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}+ req_meta = {'referrer_policy': POLICY_NO_REFERRER}+++class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}+ req_meta = {'referrer_policy': POLICY_UNSAFE_URL}+++class TestRefererMiddlewareSettingsPolicyByName(TestCase):++ def test_valid_name(self):+ for s, p in [+ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),+ (POLICY_NO_REFERRER, NoReferrerPolicy),+ (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),+ (POLICY_SAME_ORIGIN, SameOriginPolicy),+ (POLICY_ORIGIN, OriginPolicy),+ (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),+ (POLICY_UNSAFE_URL, UnsafeUrlPolicy),+ ]:+ settings = Settings({'REFERER_POLICY': s})+ mw = RefererMiddleware(settings)+ self.assertEquals(mw.default_policy, p)++ def test_invalid_name(self):+ settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'})+ with self.assertRaises(NotConfigured):+ mw = RefererMiddleware(settings)