Scrapy, a fast high-level web crawling & scraping framework for Python.
Update tests
Paul Tremberthd2aa51c0fb81
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 81868efab..39bbaab5d 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -1,3 +1,4 @@+from six.moves.urllib.parse import urlparsefrom unittest import TestCasefrom scrapy.exceptions import NotConfigured@@ -326,9 +327,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):depending on the scheme of the target URL."""def referrer(self, response, request):- from scrapy.utils.httpobj import urlparse_cached-- scheme = urlparse_cached(request).scheme+ scheme = urlparse(request).schemeif scheme == 'https':return b'https://python.org/'elif scheme == 'http':@@ -446,3 +445,41 @@ class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefereclass TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}+++class TestReferrerPolicyOnRedirect(TestRefererMiddleware):++ req_meta = {}+ resp_headers = {}+ #settings = {}+ settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}+ scenarii = [+ ( # origin+ 'http://scrapytest.org/1',++ # target + redirection+ ['http://scrapytest.org/2',+ 'http://scrapytest.org/3',],++ b'http://scrapytest.org/1', # expected initial referer+ b'http://scrapytest.org/1', # expected referer for the redirection request++ ),+ ]++ def setUp(self):+ self.spider = Spider('foo')+ settings = Settings(self.settings)+ self.mw = RefererMiddleware(settings)++ def test_(self):++ for origin, target_chain, init_referrer, final_referrer in self.scenarii:+ response = self.get_response(origin)+ request = self.get_request(target_chain.pop())+++ out = list(self.mw.process_spider_output(response, [request], self.spider))+ self.assertEquals(out[0].headers.get('Referer'), init_referrer)++ request.meta['redirected_urls'] = target_chain