Scrapy, a fast high-level web crawling & scraping framework for Python.
Update Referrer tests for "strict-" policies
Paul Tremberth5cef67ae75dc
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 9555817d9..df20dfbb9 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -8,6 +8,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \+ POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \DefaultReferrerPolicy, \NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \OriginWhenCrossOriginPolicy, OriginPolicy, \@@ -39,7 +40,6 @@ class TestRefererMiddleware(TestCase):for origin, target, referrer in self.scenarii:response = self.get_response(origin)request = self.get_request(target)-out = list(self.mw.process_spider_output(response, [request], self.spider))self.assertEquals(out[0].headers.get('Referer'), referrer)@@ -154,6 +154,25 @@ class MixinOrigin(object):]+class MixinStrictOrigin(object):+ scenarii = [+ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades+ ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),+ ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),+ ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),++ # downgrade: send nothing+ ('https://example.com/page.html', 'http://scrapy.org', None),++ # upgrade: send origin+ ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),++ # test for user/password stripping+ ('https://user:[email protected]/page.html', 'https://scrapy.org', b'https://example.com/'),+ ('https://user:[email protected]/page.html', 'http://scrapy.org', None),+ ]++class MixinOriginWhenCrossOrigin(object):scenarii = [# Same origin (protocol, host, port): send referrer@@ -189,6 +208,44 @@ class MixinOriginWhenCrossOrigin(object):]+class MixinStrictOriginWhenCrossOrigin(object):+ scenarii = [+ # Same origin (protocol, host, port): send referrer+ ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),+ ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),+ ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),+ ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),+ ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),++ # Different host: send origin as referrer+ ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),+ ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),+ ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),+ # exact match required+ ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),++ # Different port: send origin as referrer+ ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),+ ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),++ # downgrade+ ('https://example4.com/page.html', 'http://example4.com/not-page.html', None),+ ('https://example4.com/page.html', 'http://not.example4.com/', None),+ ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None),++ # Different protocols: send origin as referrer+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),++ # test for user/password stripping+ ('https://user:[email protected]/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),++ # TLS to non-TLS downgrade: send nothing+ ('https://user:[email protected]/page.html', 'http://example5.com/not-page.html', None),+ ]++class MixinUnsafeUrl(object):scenarii = [# TLS to TLS: send referrer@@ -241,10 +298,18 @@ class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}+class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}++class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}+class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):+ settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}++class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}@@ -297,10 +362,18 @@ class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_ORIGIN}+class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}++class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}+class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):+ req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}++class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_UNSAFE_URL}