Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve and simplify tests.

Andrey Rakhmatullin028a56b9a2e0
diff --git a/tests/test_addons.py b/tests/test_addons.py
index aa1b760c2..8375a6495 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -3,6 +3,8 @@ import unittest
from typing import Any, Dict
from unittest.mock import patch
+from twisted.internet.defer import inlineCallbacks
+
from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerRunner
from scrapy.exceptions import NotConfigured
@@ -177,3 +179,22 @@ class AddonManagerTest(unittest.TestCase):
{"addons": [addon]},
extra={"crawler": crawler},
)
+
+ @inlineCallbacks
+ def test_enable_addon_in_spider(self):
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ addon_config = {"KEY": "addon"}
+ addon_cls = get_addon_cls(addon_config)
+ spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider")
+ return spider
+
+ runner = CrawlerRunner({"KEY": "project"})
+ crawler = runner.create_crawler(MySpider)
+ self.assertEqual(crawler.settings.get("KEY"), "project")
+ yield crawler.crawl()
+ self.assertEqual(crawler.settings.get("KEY"), "addon")
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 496ab77a5..96d43b2b9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,7 +1,6 @@
import json
import logging
import unittest
-import warnings
from ipaddress import IPv4Address
from socket import gethostbyname
from urllib.parse import urlparse
@@ -14,14 +13,11 @@ from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from scrapy import signals
-from scrapy.crawler import Crawler, CrawlerRunner
-from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
-from scrapy.extensions.throttle import AutoThrottle
+from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
-from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
-from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import MockServer
@@ -414,38 +410,6 @@ with multiples lines
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
- @defer.inlineCallbacks
- def test_populate_spidercls_settings(self):
- spider_settings = {
- "TEST1": "spider",
- "TEST2": "spider",
- "AUTOTHROTTLE_ENABLED": True,
- }
- project_settings = {"TEST1": "project", "TEST3": "project"}
-
- class CustomSettingsSpider(DefaultSpider):
- custom_settings = spider_settings
-
- def parse(self, response):
- return
-
- settings = Settings()
- settings.setdict(project_settings, priority="project")
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- crawler = Crawler(CustomSettingsSpider, settings)
- yield crawler.crawl()
-
- self.assertEqual(crawler.settings.get("TEST1"), "spider")
- self.assertEqual(crawler.settings.get("TEST2"), "spider")
- self.assertEqual(crawler.settings.get("TEST3"), "project")
-
- enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
- self.assertIn(AutoThrottle, enabled_exts)
-
- self.assertFalse(settings.frozen)
- self.assertTrue(crawler.settings.frozen)
-
class CrawlSpiderTestCase(TestCase):
def setUp(self):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index bfae6c690..08149725c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -17,6 +17,7 @@ import scrapy
from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions import telnet
+from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
@@ -32,6 +33,25 @@ class BaseCrawlerTest(unittest.TestCase):
class CrawlerTestCase(BaseCrawlerTest):
+ def test_populate_spidercls_settings(self):
+ spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+ project_settings = {"TEST1": "project", "TEST3": "project"}
+
+ class CustomSettingsSpider(DefaultSpider):
+ custom_settings = spider_settings
+
+ settings = Settings()
+ settings.setdict(project_settings, priority="project")
+ crawler = Crawler(CustomSettingsSpider, settings)
+ crawler._load_settings()
+
+ self.assertEqual(crawler.settings.get("TEST1"), "spider")
+ self.assertEqual(crawler.settings.get("TEST2"), "spider")
+ self.assertEqual(crawler.settings.get("TEST3"), "project")
+
+ self.assertFalse(settings.frozen)
+ self.assertTrue(crawler.settings.frozen)
+
def test_crawler_accepts_dict(self):
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
self.assertEqual(crawler.settings["foo"], "bar")
@@ -58,6 +78,17 @@ class CrawlerTestCase(BaseCrawlerTest):
yield crawler.crawl()
+class SpiderSettingsTestCase(unittest.TestCase):
+ def test_spider_custom_settings(self):
+ class MySpider(scrapy.Spider):
+ name = "spider"
+ custom_settings = {"AUTOTHROTTLE_ENABLED": True}
+
+ crawler = get_crawler(MySpider)
+ enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
+ self.assertIn(AutoThrottle, enabled_exts)
+
+
class CrawlerLoggingTestCase(unittest.TestCase):
def test_no_root_handler_installed(self):
handler = get_scrapy_root_handler()