Scrapy, a fast high-level web crawling & scraping framework for Python.
Rule.process_request: tests
Eugenio Lacuesta22fda61d62a2
diff --git a/tests/test_spider.py b/tests/test_spider.pyindex fefdaa403..5e20e0d99 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -263,6 +263,104 @@ class CrawlSpiderTest(SpiderTest):'http://example.org/about.html','http://example.org/nofollow.html'])+ def test_process_request(self):++ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)++ def process_request_change_domain(request):+ return request.replace(url=request.url.replace('.org', '.com'))++ class _CrawlSpider(self.spider_class):+ name="test"+ allowed_domains=['example.org']+ rules = (+ Rule(LinkExtractor(), process_request=process_request_change_domain),+ )++ spider = _CrawlSpider()+ output = list(spider._requests_to_follow(response))+ self.assertEqual(len(output), 3)+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))+ self.assertEqual([r.url for r in output],+ ['http://example.com/somepage/item/12.html',+ 'http://example.com/about.html',+ 'http://example.com/nofollow.html'])++ def test_process_request_with_response(self):++ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)++ def process_request_meta_response_class(request, response):+ request.meta['response_class'] = response.__class__.__name__+ return request++ class _CrawlSpider(self.spider_class):+ name="test"+ allowed_domains=['example.org']+ rules = (+ Rule(LinkExtractor(), process_request=process_request_meta_response_class),+ )++ spider = _CrawlSpider()+ output = list(spider._requests_to_follow(response))+ self.assertEqual(len(output), 3)+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))+ self.assertEqual([r.url for r in output],+ ['http://example.org/somepage/item/12.html',+ 'http://example.org/about.html',+ 'http://example.org/nofollow.html'])+ self.assertEqual([r.meta['response_class'] for r in output],+ ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])++ def test_process_request_instance_method(self):++ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)++ class _CrawlSpider(self.spider_class):+ name="test"+ allowed_domains=['example.org']+ rules = (+ Rule(LinkExtractor(), process_request='process_request_upper'),+ )++ def process_request_upper(self, request):+ return request.replace(url=request.url.upper())++ spider = _CrawlSpider()+ output = list(spider._requests_to_follow(response))+ self.assertEqual(len(output), 3)+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))+ self.assertEqual([r.url for r in output],+ ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',+ 'http://EXAMPLE.ORG/ABOUT.HTML',+ 'http://EXAMPLE.ORG/NOFOLLOW.HTML'])++ def test_process_request_instance_method_with_response(self):++ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)++ class _CrawlSpider(self.spider_class):+ name="test"+ allowed_domains=['example.org']+ rules = (+ Rule(LinkExtractor(), process_request='process_request_meta_response_class'),+ )++ def process_request_meta_response_class(self, request, response):+ request.meta['response_class'] = response.__class__.__name__+ return request++ spider = _CrawlSpider()+ output = list(spider._requests_to_follow(response))+ self.assertEqual(len(output), 3)+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))+ self.assertEqual([r.url for r in output],+ ['http://example.org/somepage/item/12.html',+ 'http://example.org/about.html',+ 'http://example.org/nofollow.html'])+ self.assertEqual([r.meta['response_class'] for r in output],+ ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])+def test_follow_links_attribute_population(self):crawler = get_crawler()spider = self.spider_class.from_crawler(crawler, 'example.com')