Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

improve test_closespider_pagecount_no_item and MaxItemsSpider

guillermo-bondonnoa40d5281cfb8
diff --git a/tests/spiders.py b/tests/spiders.py
index 2bcec5624..d1998ca69 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -283,21 +283,18 @@ class ItemSpider(FollowAllSpider):
yield {}
-class MaxItemsSpider(ItemSpider):
+class MaxItemsKeepCrawlingSpider(FollowAllSpider):
def __init__(self, max_items=10, *args, **kwargs):
super().__init__(*args, **kwargs)
self.max_items = max_items
self.items_scraped = 0
def parse(self, response):
- for item_or_req in super().parse(response):
- if isinstance(item_or_req, Request):
- yield item_or_req
- else:
- if self.items_scraped >= self.max_items:
- continue
+ for request in super().parse(response):
+ yield request
+ if self.items_scraped < self.max_items:
+ yield Item()
self.items_scraped += 1
- yield item_or_req
class DefaultError(Exception):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 0046b4e29..50b483a74 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -7,7 +7,7 @@ from tests.spiders import (
ErrorSpider,
FollowAllSpider,
ItemSpider,
- MaxItemsSpider,
+ MaxItemsKeepCrawlingSpider,
SlowSpider,
)
@@ -43,13 +43,11 @@ class TestCloseSpider(TestCase):
@defer.inlineCallbacks
def test_closespider_pagecount_no_item(self):
close_on = 5
- close_on_pagecount = 20
max_items = 5
crawler = get_crawler(
- MaxItemsSpider,
+ MaxItemsKeepCrawlingSpider,
{
"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
- "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
},
)
yield crawler.crawl(max_items=max_items, mockserver=self.mockserver)
@@ -58,8 +56,8 @@ class TestCloseSpider(TestCase):
pagecount = crawler.stats.get_value("response_received_count")
itemcount = crawler.stats.get_value("item_scraped_count")
self.assertEqual(itemcount, max_items)
- self.assertLess(pagecount, close_on_pagecount)
- self.assertTrue((pagecount - itemcount) >= close_on)
+ self.assertLessEqual(pagecount, close_on + itemcount)
+ self.assertGreater(pagecount, itemcount)
@defer.inlineCallbacks
def test_closespider_errorcount(self):