Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactor EngineTest tests.
Andrey Rakhmatullin200d76afa96a
diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 95955f7be..91ce2c0de 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -243,46 +243,7 @@ class CrawlerRun:self.signals_caught[sig] = signalargs-class EngineTest(unittest.TestCase):- @defer.inlineCallbacks- def test_crawler(self):- for spider in (- TestSpider,- DictItemsSpider,- AttrsItemsSpider,- DataClassItemsSpider,- ):- run = CrawlerRun(spider)- yield run.run()- self._assert_visited_urls(run)- self._assert_scheduled_requests(run, count=9)- self._assert_downloaded_responses(run, count=9)- self._assert_scraped_items(run)- self._assert_signals_caught(run)- self._assert_bytes_received(run)-- @defer.inlineCallbacks- def test_crawler_dupefilter(self):- run = CrawlerRun(TestDupeFilterSpider)- yield run.run()- self._assert_scheduled_requests(run, count=8)- self._assert_dropped_requests(run)-- @defer.inlineCallbacks- def test_crawler_itemerror(self):- run = CrawlerRun(ItemZeroDivisionErrorSpider)- yield run.run()- self._assert_items_error(run)-- @defer.inlineCallbacks- def test_crawler_change_close_reason_on_idle(self):- run = CrawlerRun(ChangeCloseReasonSpider)- yield run.run()- self.assertEqual(- {"spider": run.spider, "reason": "custom_reason"},- run.signals_caught[signals.spider_closed],- )-+class EngineTestBase(unittest.TestCase):def _assert_visited_urls(self, run: CrawlerRun):must_be_visited = ["/",@@ -422,6 +383,47 @@ class EngineTest(unittest.TestCase):run.signals_caught[signals.spider_closed],)++class EngineTest(EngineTestBase):+ @defer.inlineCallbacks+ def test_crawler(self):+ for spider in (+ TestSpider,+ DictItemsSpider,+ AttrsItemsSpider,+ DataClassItemsSpider,+ ):+ run = CrawlerRun(spider)+ yield run.run()+ self._assert_visited_urls(run)+ self._assert_scheduled_requests(run, count=9)+ self._assert_downloaded_responses(run, count=9)+ self._assert_scraped_items(run)+ self._assert_signals_caught(run)+ self._assert_bytes_received(run)++ @defer.inlineCallbacks+ def test_crawler_dupefilter(self):+ run = CrawlerRun(TestDupeFilterSpider)+ yield run.run()+ self._assert_scheduled_requests(run, count=8)+ self._assert_dropped_requests(run)++ @defer.inlineCallbacks+ def test_crawler_itemerror(self):+ run = CrawlerRun(ItemZeroDivisionErrorSpider)+ yield run.run()+ self._assert_items_error(run)++ @defer.inlineCallbacks+ def test_crawler_change_close_reason_on_idle(self):+ run = CrawlerRun(ChangeCloseReasonSpider)+ yield run.run()+ self.assertEqual(+ {"spider": run.spider, "reason": "custom_reason"},+ run.signals_caught[signals.spider_closed],+ )+@defer.inlineCallbacksdef test_close_downloader(self):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.pyindex 8dbb5b7ea..8bf225ab1 100644--- a/tests/test_engine_stop_download_bytes.py+++ b/tests/test_engine_stop_download_bytes.py@@ -7,7 +7,7 @@ from tests.test_engine import (CrawlerRun,DataClassItemsSpider,DictItemsSpider,- EngineTest,+ EngineTestBase,TestSpider,)@@ -18,7 +18,7 @@ class BytesReceivedCrawlerRun(CrawlerRun):raise StopDownload(fail=False)-class BytesReceivedEngineTest(EngineTest):+class BytesReceivedEngineTest(EngineTestBase):@defer.inlineCallbacksdef test_crawler(self):for spider in (diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.pyindex db35bd81e..4efb6b7a8 100644--- a/tests/test_engine_stop_download_headers.py+++ b/tests/test_engine_stop_download_headers.py@@ -7,7 +7,7 @@ from tests.test_engine import (CrawlerRun,DataClassItemsSpider,DictItemsSpider,- EngineTest,+ EngineTestBase,TestSpider,)@@ -18,7 +18,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun):raise StopDownload(fail=False)-class HeadersReceivedEngineTest(EngineTest):+class HeadersReceivedEngineTest(EngineTestBase):@defer.inlineCallbacksdef test_crawler(self):for spider in (