Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update tests

Paul Tremberthd2aa51c0fb81
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 81868efab..39bbaab5d 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,3 +1,4 @@
+from six.moves.urllib.parse import urlparse
from unittest import TestCase
from scrapy.exceptions import NotConfigured
@@ -326,9 +327,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
depending on the scheme of the target URL.
"""
def referrer(self, response, request):
- from scrapy.utils.httpobj import urlparse_cached
-
- scheme = urlparse_cached(request).scheme
+ scheme = urlparse(request).scheme
if scheme == 'https':
return b'https://python.org/'
elif scheme == 'http':
@@ -446,3 +445,41 @@ class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefere
class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
+
+
+class TestReferrerPolicyOnRedirect(TestRefererMiddleware):
+
+ req_meta = {}
+ resp_headers = {}
+ #settings = {}
+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+ scenarii = [
+ ( # origin
+ 'http://scrapytest.org/1',
+
+ # target + redirection
+ ['http://scrapytest.org/2',
+ 'http://scrapytest.org/3',],
+
+ b'http://scrapytest.org/1', # expected initial referer
+ b'http://scrapytest.org/1', # expected referer for the redirection request
+
+ ),
+ ]
+
+ def setUp(self):
+ self.spider = Spider('foo')
+ settings = Settings(self.settings)
+ self.mw = RefererMiddleware(settings)
+
+ def test_(self):
+
+ for origin, target_chain, init_referrer, final_referrer in self.scenarii:
+ response = self.get_response(origin)
+ request = self.get_request(target_chain.pop())
+
+
+ out = list(self.mw.process_spider_output(response, [request], self.spider))
+ self.assertEquals(out[0].headers.get('Referer'), init_referrer)
+
+ request.meta['redirected_urls'] = target_chain