Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rule.process_request: tests

Eugenio Lacuesta22fda61d62a2
diff --git a/tests/test_spider.py b/tests/test_spider.py
index fefdaa403..5e20e0d99 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -263,6 +263,104 @@ class CrawlSpiderTest(SpiderTest):
'http://example.org/about.html',
'http://example.org/nofollow.html'])
+ def test_process_request(self):
+
+ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+ def process_request_change_domain(request):
+ return request.replace(url=request.url.replace('.org', '.com'))
+
+ class _CrawlSpider(self.spider_class):
+ name="test"
+ allowed_domains=['example.org']
+ rules = (
+ Rule(LinkExtractor(), process_request=process_request_change_domain),
+ )
+
+ spider = _CrawlSpider()
+ output = list(spider._requests_to_follow(response))
+ self.assertEqual(len(output), 3)
+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+ self.assertEqual([r.url for r in output],
+ ['http://example.com/somepage/item/12.html',
+ 'http://example.com/about.html',
+ 'http://example.com/nofollow.html'])
+
+ def test_process_request_with_response(self):
+
+ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+ def process_request_meta_response_class(request, response):
+ request.meta['response_class'] = response.__class__.__name__
+ return request
+
+ class _CrawlSpider(self.spider_class):
+ name="test"
+ allowed_domains=['example.org']
+ rules = (
+ Rule(LinkExtractor(), process_request=process_request_meta_response_class),
+ )
+
+ spider = _CrawlSpider()
+ output = list(spider._requests_to_follow(response))
+ self.assertEqual(len(output), 3)
+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+ self.assertEqual([r.url for r in output],
+ ['http://example.org/somepage/item/12.html',
+ 'http://example.org/about.html',
+ 'http://example.org/nofollow.html'])
+ self.assertEqual([r.meta['response_class'] for r in output],
+ ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+
+ def test_process_request_instance_method(self):
+
+ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+ class _CrawlSpider(self.spider_class):
+ name="test"
+ allowed_domains=['example.org']
+ rules = (
+ Rule(LinkExtractor(), process_request='process_request_upper'),
+ )
+
+ def process_request_upper(self, request):
+ return request.replace(url=request.url.upper())
+
+ spider = _CrawlSpider()
+ output = list(spider._requests_to_follow(response))
+ self.assertEqual(len(output), 3)
+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+ self.assertEqual([r.url for r in output],
+ ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+ 'http://EXAMPLE.ORG/ABOUT.HTML',
+ 'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+
+ def test_process_request_instance_method_with_response(self):
+
+ response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+ class _CrawlSpider(self.spider_class):
+ name="test"
+ allowed_domains=['example.org']
+ rules = (
+ Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
+ )
+
+ def process_request_meta_response_class(self, request, response):
+ request.meta['response_class'] = response.__class__.__name__
+ return request
+
+ spider = _CrawlSpider()
+ output = list(spider._requests_to_follow(response))
+ self.assertEqual(len(output), 3)
+ self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+ self.assertEqual([r.url for r in output],
+ ['http://example.org/somepage/item/12.html',
+ 'http://example.org/about.html',
+ 'http://example.org/nofollow.html'])
+ self.assertEqual([r.meta['response_class'] for r in output],
+ ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+
def test_follow_links_attribute_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, 'example.com')