Scrapy, a fast high-level web crawling & scraping framework for Python.
improve tests by having SlowSpider
Kevin Lloyd Bernal368ab29ffc32
diff --git a/tests/spiders.py b/tests/spiders.pyindex 6ff48f471..f29dea2a1 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -77,6 +77,22 @@ class DelaySpider(MetaSpider):self.t2_err = time.time()+class SlowSpider(DelaySpider):+ name = "slow"++ def start_requests(self):+ # 1st response is fast+ url = self.mockserver.url("/delay?n=0&b=0")+ yield Request(url, callback=self.parse, errback=self.errback)++ # 2nd response is slow+ url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")+ yield Request(url, callback=self.parse, errback=self.errback)++ def parse(self, response):+ yield Item()++class SimpleSpider(MetaSpider):name = "simple"diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 259a1a4ad..38ede70e4 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCasefrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer-from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider+from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpiderclass TestCloseSpider(TestCase):@@ -58,8 +58,8 @@ class TestCloseSpider(TestCase):@defer.inlineCallbacksdef test_closespider_timeout_no_item(self):timeout = 1- crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})- yield crawler.crawl(n=3, total=10, mockserver=self.mockserver)+ crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})+ yield crawler.crawl(n=3, mockserver=self.mockserver)reason = crawler.spider.meta["close_reason"]self.assertEqual(reason, "closespider_timeout_no_item")total_seconds = crawler.stats.get_value("elapsed_time_seconds")