Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

improve tests by having SlowSpider

Kevin Lloyd Bernal368ab29ffc32
diff --git a/tests/spiders.py b/tests/spiders.py
index 6ff48f471..f29dea2a1 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -77,6 +77,22 @@ class DelaySpider(MetaSpider):
self.t2_err = time.time()
+class SlowSpider(DelaySpider):
+ name = "slow"
+
+ def start_requests(self):
+ # 1st response is fast
+ url = self.mockserver.url("/delay?n=0&b=0")
+ yield Request(url, callback=self.parse, errback=self.errback)
+
+ # 2nd response is slow
+ url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
+ yield Request(url, callback=self.parse, errback=self.errback)
+
+ def parse(self, response):
+ yield Item()
+
+
class SimpleSpider(MetaSpider):
name = "simple"
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 259a1a4ad..38ede70e4 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider
+from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider
class TestCloseSpider(TestCase):
@@ -58,8 +58,8 @@ class TestCloseSpider(TestCase):
@defer.inlineCallbacks
def test_closespider_timeout_no_item(self):
timeout = 1
- crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
- yield crawler.crawl(n=3, total=10, mockserver=self.mockserver)
+ crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
+ yield crawler.crawl(n=3, mockserver=self.mockserver)
reason = crawler.spider.meta["close_reason"]
self.assertEqual(reason, "closespider_timeout_no_item")
total_seconds = crawler.stats.get_value("elapsed_time_seconds")