Scrapy, a fast high-level web crawling & scraping framework for Python.
improve test_closespider_pagecount_no_item and MaxItemsSpider
guillermo-bondonnoa40d5281cfb8
diff --git a/tests/spiders.py b/tests/spiders.pyindex 2bcec5624..d1998ca69 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -283,21 +283,18 @@ class ItemSpider(FollowAllSpider):yield {}-class MaxItemsSpider(ItemSpider):+class MaxItemsKeepCrawlingSpider(FollowAllSpider):def __init__(self, max_items=10, *args, **kwargs):super().__init__(*args, **kwargs)self.max_items = max_itemsself.items_scraped = 0def parse(self, response):- for item_or_req in super().parse(response):- if isinstance(item_or_req, Request):- yield item_or_req- else:- if self.items_scraped >= self.max_items:- continue+ for request in super().parse(response):+ yield request+ if self.items_scraped < self.max_items:+ yield Item()self.items_scraped += 1- yield item_or_reqclass DefaultError(Exception):diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 0046b4e29..50b483a74 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -7,7 +7,7 @@ from tests.spiders import (ErrorSpider,FollowAllSpider,ItemSpider,- MaxItemsSpider,+ MaxItemsKeepCrawlingSpider,SlowSpider,)@@ -43,13 +43,11 @@ class TestCloseSpider(TestCase):@defer.inlineCallbacksdef test_closespider_pagecount_no_item(self):close_on = 5- close_on_pagecount = 20max_items = 5crawler = get_crawler(- MaxItemsSpider,+ MaxItemsKeepCrawlingSpider,{"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,- "CLOSESPIDER_PAGECOUNT": close_on_pagecount,},)yield crawler.crawl(max_items=max_items, mockserver=self.mockserver)@@ -58,8 +56,8 @@ class TestCloseSpider(TestCase):pagecount = crawler.stats.get_value("response_received_count")itemcount = crawler.stats.get_value("item_scraped_count")self.assertEqual(itemcount, max_items)- self.assertLess(pagecount, close_on_pagecount)- self.assertTrue((pagecount - itemcount) >= close_on)+ self.assertLessEqual(pagecount, close_on + itemcount)+ self.assertGreater(pagecount, itemcount)@defer.inlineCallbacksdef test_closespider_errorcount(self):