Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve and simplify tests.
Andrey Rakhmatullin028a56b9a2e0
diff --git a/tests/test_addons.py b/tests/test_addons.pyindex aa1b760c2..8375a6495 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -3,6 +3,8 @@ import unittestfrom typing import Any, Dictfrom unittest.mock import patch+from twisted.internet.defer import inlineCallbacks+from scrapy import Spiderfrom scrapy.crawler import Crawler, CrawlerRunnerfrom scrapy.exceptions import NotConfigured@@ -177,3 +179,22 @@ class AddonManagerTest(unittest.TestCase):{"addons": [addon]},extra={"crawler": crawler},)++ @inlineCallbacks+ def test_enable_addon_in_spider(self):+ class MySpider(Spider):+ name = "myspider"++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ addon_config = {"KEY": "addon"}+ addon_cls = get_addon_cls(addon_config)+ spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider")+ return spider++ runner = CrawlerRunner({"KEY": "project"})+ crawler = runner.create_crawler(MySpider)+ self.assertEqual(crawler.settings.get("KEY"), "project")+ yield crawler.crawl()+ self.assertEqual(crawler.settings.get("KEY"), "addon")diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 496ab77a5..96d43b2b9 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,7 +1,6 @@import jsonimport loggingimport unittest-import warningsfrom ipaddress import IPv4Addressfrom socket import gethostbynamefrom urllib.parse import urlparse@@ -14,14 +13,11 @@ from twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom scrapy import signals-from scrapy.crawler import Crawler, CrawlerRunner-from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload-from scrapy.extensions.throttle import AutoThrottle+from scrapy.crawler import CrawlerRunner+from scrapy.exceptions import StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Response-from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode-from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver import MockServer@@ -414,38 +410,6 @@ with multiples linesself._assert_retried(log)self.assertIn("Got response 200", str(log))- @defer.inlineCallbacks- def test_populate_spidercls_settings(self):- spider_settings = {- "TEST1": "spider",- "TEST2": "spider",- "AUTOTHROTTLE_ENABLED": True,- }- project_settings = {"TEST1": "project", "TEST3": "project"}-- class CustomSettingsSpider(DefaultSpider):- custom_settings = spider_settings-- def parse(self, response):- return-- settings = Settings()- settings.setdict(project_settings, priority="project")- with warnings.catch_warnings():- warnings.simplefilter("ignore", ScrapyDeprecationWarning)- crawler = Crawler(CustomSettingsSpider, settings)- yield crawler.crawl()-- self.assertEqual(crawler.settings.get("TEST1"), "spider")- self.assertEqual(crawler.settings.get("TEST2"), "spider")- self.assertEqual(crawler.settings.get("TEST3"), "project")-- enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]- self.assertIn(AutoThrottle, enabled_exts)-- self.assertFalse(settings.frozen)- self.assertTrue(crawler.settings.frozen)-class CrawlSpiderTestCase(TestCase):def setUp(self):diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex bfae6c690..08149725c 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -17,6 +17,7 @@ import scrapyfrom scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunnerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions import telnet+from scrapy.extensions.throttle import AutoThrottlefrom scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoaderfrom scrapy.utils.log import configure_logging, get_scrapy_root_handler@@ -32,6 +33,25 @@ class BaseCrawlerTest(unittest.TestCase):class CrawlerTestCase(BaseCrawlerTest):+ def test_populate_spidercls_settings(self):+ spider_settings = {"TEST1": "spider", "TEST2": "spider"}+ project_settings = {"TEST1": "project", "TEST3": "project"}++ class CustomSettingsSpider(DefaultSpider):+ custom_settings = spider_settings++ settings = Settings()+ settings.setdict(project_settings, priority="project")+ crawler = Crawler(CustomSettingsSpider, settings)+ crawler._load_settings()++ self.assertEqual(crawler.settings.get("TEST1"), "spider")+ self.assertEqual(crawler.settings.get("TEST2"), "spider")+ self.assertEqual(crawler.settings.get("TEST3"), "project")++ self.assertFalse(settings.frozen)+ self.assertTrue(crawler.settings.frozen)+def test_crawler_accepts_dict(self):crawler = get_crawler(DefaultSpider, {"foo": "bar"})self.assertEqual(crawler.settings["foo"], "bar")@@ -58,6 +78,17 @@ class CrawlerTestCase(BaseCrawlerTest):yield crawler.crawl()+class SpiderSettingsTestCase(unittest.TestCase):+ def test_spider_custom_settings(self):+ class MySpider(scrapy.Spider):+ name = "spider"+ custom_settings = {"AUTOTHROTTLE_ENABLED": True}++ crawler = get_crawler(MySpider)+ enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]+ self.assertIn(AutoThrottle, enabled_exts)++class CrawlerLoggingTestCase(unittest.TestCase):def test_no_root_handler_installed(self):handler = get_scrapy_root_handler()