Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve the performance of the DOWNLOAD_DELAY test
Adrián Chaves058bdda0afe9
diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 3fc13eeb7..a524287eb 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -30,25 +30,44 @@ class CrawlTestCase(TestCase):self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url@defer.inlineCallbacks- def test_delay(self):- # short to long delays- yield self._test_delay(0.2, False)- yield self._test_delay(1, False)- # randoms- yield self._test_delay(0.2, True)- yield self._test_delay(1, True)+ def test_fixed_delay(self):+ yield self._test_delay(total=3, delay=0.1)@defer.inlineCallbacks- def _test_delay(self, delay, randomize):- settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}+ def test_randomized_delay(self):+ yield self._test_delay(total=3, delay=0.1, randomize=True)++ @defer.inlineCallbacks+ def _test_delay(self, total, delay, randomize=False):+ crawl_kwargs = dict(+ maxlatency=delay * 2,+ mockserver=self.mockserver,+ total=total,+ )+ tolerance = (1 - (0.6 if randomize else 0.2))++ settings = {"DOWNLOAD_DELAY": delay,+ 'RANDOMIZE_DOWNLOAD_DELAY': randomize}+ crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)+ yield crawler.crawl(**crawl_kwargs)+ times = crawler.spider.times+ total_time = times[-1] - times[0]+ average = total_time / (len(times) - 1)+ self.assertTrue(average > delay * tolerance,+ "download delay too small: %s" % average)++ # Ensure that the same test parameters would cause a failure if no+ # download delay is set. Otherwise, it means we are using a combination+ # of ``total`` and ``delay`` values that are too small for the test+ # code above to have any meaning.+ settings["DOWNLOAD_DELAY"] = 0crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)- yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver)- t = crawler.spider.times- totaltime = t[-1] - t[0]- avgd = totaltime / (len(t) - 1)- tolerance = 0.6 if randomize else 0.2- self.assertTrue(avgd > delay * (1 - tolerance),- "download delay too small: %s" % avgd)+ yield crawler.crawl(**crawl_kwargs)+ times = crawler.spider.times+ total_time = times[-1] - times[0]+ average = total_time / (len(times) - 1)+ self.assertFalse(average > delay / tolerance,+ "test total or delay values are too small")@defer.inlineCallbacksdef test_timeout_success(self):