Scrapy, a fast high-level web crawling & scraping framework for Python.
Add Crawler._load_settings().
Andrey Rakhmatullinbb15c93a2bbd
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 0a5e61f7a..71f43365d 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -77,6 +77,7 @@ class Command(ScrapyCommand):# The crawler is created this way since the Shell manually handles the# crawling engine, so the set up in the crawl method won't workcrawler = self.crawler_process._create_crawler(spidercls)+ crawler._load_settings()# The Shell class needs a persistent engine in the crawlercrawler.engine = crawler._create_engine()crawler.engine.start()diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 49034c9f1..67e44541d 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -82,6 +82,7 @@ class Crawler:self._init_reactor: bool = init_reactorself.crawling: bool = False+ self._settings_loaded: bool = Falseself._started: bool = Falseself.extensions: Optional[ExtensionManager] = Noneself.stats: Optional[StatsCollector] = None@@ -90,6 +91,53 @@ class Crawler:self.spider: Optional[Spider] = Noneself.engine: Optional[ExecutionEngine] = None+ def _load_settings(self) -> None:+ if self._settings_loaded:+ return+ self._settings_loaded = True++ self.addons.load_settings(self.settings)+ self.stats = load_object(self.settings["STATS_CLASS"])(self)++ handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))+ logging.root.addHandler(handler)+ # lambda is assigned to Crawler attribute because this way it is not+ # garbage collected after leaving the scope+ self.__remove_handler = lambda: logging.root.removeHandler(handler)+ self.signals.connect(self.__remove_handler, signals.engine_stopped)++ lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])+ self.logformatter = lf_cls.from_crawler(self)++ self.request_fingerprinter = create_instance(+ load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),+ settings=self.settings,+ crawler=self,+ )++ reactor_class: str = self.settings["TWISTED_REACTOR"]+ event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]+ if self._init_reactor:+ # this needs to be done after the spider settings are merged,+ # but before something imports twisted.internet.reactor+ if reactor_class:+ install_reactor(reactor_class, event_loop)+ else:+ from twisted.internet import reactor # noqa: F401+ log_reactor_info()+ if reactor_class:+ verify_installed_reactor(reactor_class)+ if is_asyncio_reactor_installed() and event_loop:+ verify_installed_asyncio_event_loop(event_loop)++ self.extensions = ExtensionManager.from_crawler(self)+ self.settings.freeze()++ d = dict(overridden_settings(self.settings))+ logger.info(+ "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}+ )+@inlineCallbacksdef crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:if self.crawling:@@ -104,49 +152,7 @@ class Crawler:try:self.spider = self._create_spider(*args, **kwargs)-- self.addons.load_settings(self.settings)- self.stats = load_object(self.settings["STATS_CLASS"])(self)-- handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))- logging.root.addHandler(handler)- # lambda is assigned to Crawler attribute because this way it is not- # garbage collected after leaving the scope- self.__remove_handler = lambda: logging.root.removeHandler(handler)- self.signals.connect(self.__remove_handler, signals.engine_stopped)-- lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])- self.logformatter = lf_cls.from_crawler(self)-- self.request_fingerprinter = create_instance(- load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),- settings=self.settings,- crawler=self,- )-- reactor_class: str = self.settings["TWISTED_REACTOR"]- event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]- if self._init_reactor:- # this needs to be done after the spider settings are merged,- # but before something imports twisted.internet.reactor- if reactor_class:- install_reactor(reactor_class, event_loop)- else:- from twisted.internet import reactor # noqa: F401- log_reactor_info()- if reactor_class:- verify_installed_reactor(reactor_class)- if is_asyncio_reactor_installed() and event_loop:- verify_installed_asyncio_event_loop(event_loop)-- self.extensions = ExtensionManager.from_crawler(self)- self.settings.freeze()-- d = dict(overridden_settings(self.settings))- logger.info(- "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}- )-+ self._load_settings()self.engine = self._create_engine()start_requests = iter(self.spider.start_requests())yield self.engine.open_spider(self.spider, start_requests)diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 44a30dc15..9397e78b9 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -69,28 +69,30 @@ def get_ftp_content_and_delete(return b"".join(ftp_data)+class TestSpider(Spider):+ name = "test"++def get_crawler(spidercls: Optional[Type[Spider]] = None,settings_dict: Optional[Dict[str, Any]] = None,prevent_warnings: bool = True,- disable_telnet: bool = True,) -> Crawler:"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project levelpriority."""from scrapy.crawler import CrawlerRunner- from scrapy.spiders import Spider# Set by default settings that prevent deprecation warnings.settings: Dict[str, Any] = {}if prevent_warnings:settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7"- if disable_telnet:- settings["TELNETCONSOLE_ENABLED"] = Falsesettings.update(settings_dict or {})runner = CrawlerRunner(settings)- return runner.create_crawler(spidercls or Spider)+ crawler = runner.create_crawler(spidercls or TestSpider)+ crawler._load_settings()+ return crawlerdef get_pythonpath() -> str:diff --git a/tests/spiders.py b/tests/spiders.pyindex eeb0194eb..f29dea2a1 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -18,13 +18,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req-class NoRequestsSpider(Spider):- name = "no_request"-- def start_requests(self):- return []--class MockServerSpider(Spider):def __init__(self, mockserver=None, *args, **kwargs):super().__init__(*args, **kwargs)diff --git a/tests/test_addons.py b/tests/test_addons.pyindex c4b4b7ac9..aa1b760c2 100644--- a/tests/test_addons.py+++ b/tests/test_addons.py@@ -1,15 +1,13 @@import itertools+import unittestfrom typing import Any, Dictfrom unittest.mock import patch-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittest-+from scrapy import Spiderfrom scrapy.crawler import Crawler, CrawlerRunnerfrom scrapy.exceptions import NotConfiguredfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpiderclass SimpleAddon:@@ -53,17 +51,14 @@ class AddonTest(unittest.TestCase):class AddonManagerTest(unittest.TestCase):- @inlineCallbacksdef test_load_settings(self):settings_dict = {"ADDONS": {"tests.test_addons.SimpleAddon": 0},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)manager = crawler.addonsself.assertIsInstance(manager.addons[0], SimpleAddon)- @inlineCallbacksdef test_notconfigured(self):class NotConfiguredAddon:def update_settings(self, settings):@@ -72,12 +67,10 @@ class AddonManagerTest(unittest.TestCase):settings_dict = {"ADDONS": {NotConfiguredAddon: 0},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)manager = crawler.addonsself.assertFalse(manager.addons)- @inlineCallbacksdef test_load_settings_order(self):# Get three addons with different settingsaddonlist = []@@ -89,25 +82,21 @@ class AddonManagerTest(unittest.TestCase):for ordered_addons in itertools.permutations(addonlist):expected_order = [a.number for a in ordered_addons]settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)manager = crawler.addonsself.assertEqual([a.number for a in manager.addons], expected_order)self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1])- @inlineCallbacksdef test_create_instance(self):settings_dict = {"ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},"MYADDON": {"MYADDON_KEY": "val"},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)manager = crawler.addonsself.assertIsInstance(manager.addons[0], CreateInstanceAddon)self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")- @inlineCallbacksdef test_settings_priority(self):config = {"KEY": 15, # priority=addon@@ -115,15 +104,14 @@ class AddonManagerTest(unittest.TestCase):settings_dict = {"ADDONS": {get_addon_cls(config): 1},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)self.assertEqual(crawler.settings.getint("KEY"), 15)settings = Settings(settings_dict)settings.set("KEY", 0, priority="default")runner = CrawlerRunner(settings)- crawler = runner.create_crawler(NoRequestsSpider)- yield crawler.crawl()+ crawler = runner.create_crawler(Spider)+ crawler._load_settings()self.assertEqual(crawler.settings.getint("KEY"), 15)settings_dict = {@@ -133,11 +121,9 @@ class AddonManagerTest(unittest.TestCase):settings = Settings(settings_dict)settings.set("KEY", 0, priority="default")runner = CrawlerRunner(settings)- crawler = runner.create_crawler(NoRequestsSpider)- yield crawler.crawl()+ crawler = runner.create_crawler(Spider)self.assertEqual(crawler.settings.getint("KEY"), 20)- @inlineCallbacksdef test_fallback_workflow(self):FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"@@ -154,8 +140,7 @@ class AddonManagerTest(unittest.TestCase):settings_dict = {"ADDONS": {AddonWithFallback: 1},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)self.assertEqual(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler")@@ -168,14 +153,12 @@ class AddonManagerTest(unittest.TestCase):"ADDONS": {AddonWithFallback: 1},"DOWNLOAD_HANDLERS": {"https": "UserHandler"},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)self.assertEqual(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler")self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler")- @inlineCallbacksdef test_logging_message(self):class LoggedAddon:def update_settings(self, settings):@@ -188,8 +171,7 @@ class AddonManagerTest(unittest.TestCase):}addon = LoggedAddon()create_instance_mock.return_value = addon- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings_dict)logger_mock.info.assert_called_once_with("Enabled addons:\n%(addons)s",{"addons": [addon]},diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 72d06deab..6589381f3 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -7,8 +7,6 @@ from scrapy.utils.testproc import ProcessTestfrom scrapy.utils.testsite import SiteTestfrom tests import NON_EXISTING_RESOLVABLE, tests_datadir-raise unittest.SkipTest("Broken for now")-class ShellTest(ProcessTest, SiteTest, unittest.TestCase):command = "shell"diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 05be33c73..b1d7be628 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -26,8 +26,7 @@ from scrapy.commands.startproject import IGNOREfrom scrapy.settings import Settingsfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_testenv-from tests.spiders import NoRequestsSpider-from tests.test_crawler import ExceptionSpider+from tests.test_crawler import ExceptionSpider, NoRequestsSpiderclass CommandSettings(unittest.TestCase):@@ -713,7 +712,7 @@ class BadSpider(scrapy.Spider):def test_run_good_spider(self):proc, _, _ = self.runspider(- "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider)+ "import scrapy\n" + inspect.getsource(NoRequestsSpider))ret = proc.returncodeself.assertEqual(ret, 0)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex f962cecc8..120991ae7 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -10,7 +10,6 @@ import pytestfrom packaging.version import parse as parse_versionfrom pytest import mark, raisesfrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom twisted.trial import unittestfrom w3lib import __version__ as w3lib_version@@ -24,7 +23,6 @@ from scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer, get_mockserver_env-from tests.spiders import NoRequestsSpiderclass BaseCrawlerTest(unittest.TestCase):@@ -72,7 +70,6 @@ class CrawlerLoggingTestCase(unittest.TestCase):get_crawler(MySpider)assert get_scrapy_root_handler() is None- @inlineCallbacksdef test_spider_custom_settings_log_level(self):log_file = Path(self.mktemp())log_file.write_text("previous message\n", encoding="utf-8")@@ -82,20 +79,20 @@ class CrawlerLoggingTestCase(unittest.TestCase):custom_settings = {"LOG_LEVEL": "INFO","LOG_FILE": str(log_file),+ # settings to avoid extra warnings+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",+ "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,}- def start_requests(self):- logging.debug("debug message")- logging.info("info message")- logging.warning("warning message")- logging.error("error message")- return []-configure_logging()self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)crawler = get_crawler(MySpider)- yield crawler.crawl()self.assertEqual(get_scrapy_root_handler().level, logging.INFO)+ info_count = crawler.stats.get_value("log_count/INFO")+ logging.debug("debug message")+ logging.info("info message")+ logging.warning("warning message")+ logging.error("error message")logged = log_file.read_text(encoding="utf-8")@@ -106,7 +103,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):self.assertIn("error message", logged)self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1)self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1)- # self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) TODO+ self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1)self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0)def test_spider_custom_settings_log_append(self):@@ -183,6 +180,13 @@ class ExceptionSpider(scrapy.Spider):raise ValueError("Exception in from_crawler method")+class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ def start_requests(self):+ return []++@mark.usefixtures("reactor_pytest")class CrawlerRunnerHasSpider(unittest.TestCase):def _runner(self):diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex d64651211..062e8a8b4 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -3,33 +3,33 @@ from unittest import mockfrom pytest import markfrom twisted.internet import defer-from twisted.internet.defer import Deferred, inlineCallbacks+from twisted.internet.defer import Deferredfrom twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.exceptions import _InvalidOutputfrom scrapy.http import Request, Response+from scrapy.spiders import Spiderfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler, get_from_asyncio_queue-from tests.spiders import NoRequestsSpiderclass ManagerTestCase(TestCase):settings_dict = None- @inlineCallbacksdef setUp(self):- self.crawler = get_crawler(NoRequestsSpider, self.settings_dict)- yield self.crawler.crawl()- self.spider = self.crawler.spider+ self.crawler = get_crawler(Spider, self.settings_dict)+ self.spider = self.crawler._create_spider("foo")self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler)- yield self.mwman.open_spider(self.spider)+ # some mw depends on stats collector+ self.crawler.stats.open_spider(self.spider)+ return self.mwman.open_spider(self.spider)def tearDown(self):+ self.crawler.stats.close_spider(self.spider, "")return self.mwman.close_spider(self.spider)- @inlineCallbacksdef _download(self, request, response=None):"""Executes downloader mw manager's download method and returnsthe result (Request or Response) or raise exception in case of@@ -41,21 +41,26 @@ class ManagerTestCase(TestCase):def download_func(**kwargs):return response- ret = yield self.mwman.download(download_func, request, self.spider)+ dfd = self.mwman.download(download_func, request, self.spider)+ # catch deferred result and return the value+ results = []+ dfd.addBoth(results.append)+ self._wait(dfd)+ ret = results[0]+ if isinstance(ret, Failure):+ ret.raiseException()return retclass DefaultsTest(ManagerTestCase):"""Tests default behavior with default settings"""- @inlineCallbacksdef test_request_response(self):req = Request("http://example.com/index.html")resp = Response(req.url, status=200)- ret = yield self._download(req, resp)+ ret = self._download(req, resp)self.assertTrue(isinstance(ret, Response), "Non-response returned")- @inlineCallbacksdef test_3xx_and_invalid_gzipped_body_must_redirect(self):"""Regression test for a failure when redirecting a compressedrequest.@@ -81,7 +86,7 @@ class DefaultsTest(ManagerTestCase):"Location": "http://example.com/login",},)- ret = yield self._download(request=req, response=resp)+ ret = self._download(request=req, response=resp)self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}")self.assertEqual(to_bytes(ret.url),@@ -89,7 +94,6 @@ class DefaultsTest(ManagerTestCase):"Not redirected to location header",)- @inlineCallbacksdef test_200_and_invalid_gzipped_body_must_fail(self):req = Request("http://example.com")body = b"<p>You are being redirected</p>"@@ -104,14 +108,12 @@ class DefaultsTest(ManagerTestCase):"Location": "http://example.com/login",},)- with self.assertRaises(OSError):- yield self._download(request=req, response=resp)+ self.assertRaises(OSError, self._download, request=req, response=resp)class ResponseFromProcessRequestTest(ManagerTestCase):"""Tests middleware returning a response from process_request."""- @inlineCallbacksdef test_download_func_not_called(self):resp = Response("http://example.com/index.html")@@ -123,8 +125,12 @@ class ResponseFromProcessRequestTest(ManagerTestCase):req = Request("http://example.com/index.html")download_func = mock.MagicMock()- result = yield self.mwman.download(download_func, req, self.spider)- self.assertIs(result, resp)+ dfd = self.mwman.download(download_func, req, self.spider)+ results = []+ dfd.addBoth(results.append)+ self._wait(dfd)++ self.assertIs(results[0], resp)self.assertFalse(download_func.called)@@ -191,7 +197,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):class MiddlewareUsingDeferreds(ManagerTestCase):"""Middlewares using Deferreds should work"""- @inlineCallbacksdef test_deferred(self):resp = Response("http://example.com/index.html")@@ -208,8 +213,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase):self.mwman._add_middleware(DeferredMiddleware())req = Request("http://example.com/index.html")download_func = mock.MagicMock()- result = yield self.mwman.download(download_func, req, self.spider)- self.assertIs(result, resp)+ dfd = self.mwman.download(download_func, req, self.spider)+ results = []+ dfd.addBoth(results.append)+ self._wait(dfd)++ self.assertIs(results[0], resp)self.assertFalse(download_func.called)@@ -217,7 +226,6 @@ class MiddlewareUsingDeferreds(ManagerTestCase):class MiddlewareUsingCoro(ManagerTestCase):"""Middlewares using asyncio coroutines should work"""- @inlineCallbacksdef test_asyncdef(self):resp = Response("http://example.com/index.html")@@ -229,12 +237,15 @@ class MiddlewareUsingCoro(ManagerTestCase):self.mwman._add_middleware(CoroMiddleware())req = Request("http://example.com/index.html")download_func = mock.MagicMock()- result = yield self.mwman.download(download_func, req, self.spider)- self.assertIs(result, resp)+ dfd = self.mwman.download(download_func, req, self.spider)+ results = []+ dfd.addBoth(results.append)+ self._wait(dfd)++ self.assertIs(results[0], resp)self.assertFalse(download_func.called)@mark.only_asyncio()- @inlineCallbacksdef test_asyncdef_asyncio(self):resp = Response("http://example.com/index.html")@@ -247,6 +258,10 @@ class MiddlewareUsingCoro(ManagerTestCase):self.mwman._add_middleware(CoroMiddleware())req = Request("http://example.com/index.html")download_func = mock.MagicMock()- result = yield self.mwman.download(download_func, req, self.spider)- self.assertIs(result, resp)+ dfd = self.mwman.download(download_func, req, self.spider)+ results = []+ dfd.addBoth(results.append)+ self._wait(dfd)++ self.assertIs(results[0], resp)self.assertFalse(download_func.called)diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex a8a687da7..f80eff3e6 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -2,31 +2,27 @@ import email.utilsimport shutilimport tempfileimport time+import unittestfrom contextlib import contextmanager-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittest-from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddlewarefrom scrapy.exceptions import IgnoreRequestfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.settings import Settings+from scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpiderclass _BaseTest(unittest.TestCase):storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"policy_class = "scrapy.extensions.httpcache.RFC2616Policy"- @inlineCallbacksdef setUp(self):self.yesterday = email.utils.formatdate(time.time() - 86400)self.today = email.utils.formatdate()self.tomorrow = email.utils.formatdate(time.time() + 86400)- self.crawler = get_crawler(NoRequestsSpider)- yield self.crawler.crawl()- self.spider = self.crawler.spider+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider("example.com")self.tmpdir = tempfile.mkdtemp()self.request = Request("http://www.example.com", headers={"User-Agent": "test"})self.response = Response(@@ -35,6 +31,7 @@ class _BaseTest(unittest.TestCase):body=b"test body",status=202,)+ self.crawler.stats.open_spider(self.spider)def tearDown(self):self.crawler.stats.close_spider(self.spider, "")@@ -569,3 +566,7 @@ class RFC2616PolicyTest(DefaultStorageTest):res2 = self._process_requestresponse(mw, req0, None)self.assertEqualResponse(res1, res2)assert "cached" in res2.flags+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 4a579c061..9dad056de 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -1,10 +1,9 @@from gzip import GzipFilefrom io import BytesIOfrom pathlib import Path+from unittest import SkipTest, TestCasefrom warnings import catch_warnings-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittestfrom w3lib.encoding import resolve_encodingfrom scrapy.downloadermiddlewares.httpcompression import (@@ -14,10 +13,10 @@ from scrapy.downloadermiddlewares.httpcompression import (from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypes+from scrapy.spiders import Spiderfrom scrapy.utils.gz import gunzipfrom scrapy.utils.test import get_crawlerfrom tests import tests_datadir-from tests.spiders import NoRequestsSpiderSAMPLEDIR = Path(tests_datadir, "compressed")@@ -39,13 +38,12 @@ FORMAT = {}-class HttpCompressionTest(unittest.TestCase):- @inlineCallbacks+class HttpCompressionTest(TestCase):def setUp(self):- self.crawler = get_crawler(NoRequestsSpider)- yield self.crawler.crawl()- self.spider = self.crawler.spider+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider("scrapytest.org")self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)+ self.crawler.stats.open_spider(self.spider)def _getresponse(self, coding):if coding not in FORMAT:@@ -133,7 +131,7 @@ class HttpCompressionTest(unittest.TestCase):try:import brotli # noqa: F401except ImportError:- raise unittest.SkipTest("no brotli")+ raise SkipTest("no brotli")response = self._getresponse("br")request = response.requestself.assertEqual(response.headers["Content-Encoding"], b"br")@@ -148,7 +146,7 @@ class HttpCompressionTest(unittest.TestCase):try:import zstandard # noqa: F401except ImportError:- raise unittest.SkipTest("no zstd support (zstandard)")+ raise SkipTest("no zstd support (zstandard)")raw_content = Nonefor check_key in FORMAT:if not check_key.startswith("zstd-"):@@ -376,15 +374,13 @@ class HttpCompressionTest(unittest.TestCase):self.assertStatsEqual("httpcompression/response_bytes", None)-class HttpCompressionSubclassTest(unittest.TestCase):- @inlineCallbacks+class HttpCompressionSubclassTest(TestCase):def test_init_missing_stats(self):class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):def __init__(self):super().__init__()- crawler = get_crawler(NoRequestsSpider)- yield crawler.crawl()+ crawler = get_crawler(Spider)with catch_warnings(record=True) as caught_warnings:HttpCompressionMiddlewareSubclass.from_crawler(crawler)messages = tuple(diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex f94958ff7..97ae1e29a 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -1,9 +1,9 @@import logging+import unittestimport warningsfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom twisted.internet.error import (ConnectError,ConnectionDone,@@ -11,7 +11,6 @@ from twisted.internet.error import (DNSLookupError,TCPTimedOutError,)-from twisted.trial import unittestfrom twisted.web.client import ResponseFailedfrom scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request@@ -20,15 +19,12 @@ from scrapy.http import Request, Responsefrom scrapy.settings.default_settings import RETRY_EXCEPTIONSfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpiderclass RetryTest(unittest.TestCase):- @inlineCallbacksdef setUp(self):- self.crawler = get_crawler(NoRequestsSpider)- yield self.crawler.crawl()- self.spider = self.crawler.spider+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider("foo")self.mw = RetryMiddleware.from_crawler(self.crawler)self.mw.max_retry_times = 2@@ -164,19 +160,16 @@ class RetryTest(unittest.TestCase):class MaxRetryTimesTest(unittest.TestCase):invalid_url = "http://www.scrapytest.org/invalid_url"- @inlineCallbacksdef get_spider_and_middleware(self, settings=None):- crawler = get_crawler(NoRequestsSpider, settings or {})- yield crawler.crawl()- spider = crawler.spider+ crawler = get_crawler(Spider, settings or {})+ spider = crawler._create_spider("foo")middleware = RetryMiddleware.from_crawler(crawler)return spider, middleware- @inlineCallbacksdef test_with_settings_zero(self):max_retry_times = 0settings = {"RETRY_TIMES": max_retry_times}- spider, middleware = yield self.get_spider_and_middleware(settings)+ spider, middleware = self.get_spider_and_middleware(settings)req = Request(self.invalid_url)self._test_retry(req,@@ -186,10 +179,9 @@ class MaxRetryTimesTest(unittest.TestCase):middleware=middleware,)- @inlineCallbacksdef test_with_metakey_zero(self):max_retry_times = 0- spider, middleware = yield self.get_spider_and_middleware()+ spider, middleware = self.get_spider_and_middleware()meta = {"max_retry_times": max_retry_times}req = Request(self.invalid_url, meta=meta)self._test_retry(@@ -200,11 +192,10 @@ class MaxRetryTimesTest(unittest.TestCase):middleware=middleware,)- @inlineCallbacksdef test_without_metakey(self):max_retry_times = 5settings = {"RETRY_TIMES": max_retry_times}- spider, middleware = yield self.get_spider_and_middleware(settings)+ spider, middleware = self.get_spider_and_middleware(settings)req = Request(self.invalid_url)self._test_retry(req,@@ -214,7 +205,6 @@ class MaxRetryTimesTest(unittest.TestCase):middleware=middleware,)- @inlineCallbacksdef test_with_metakey_greater(self):meta_max_retry_times = 3middleware_max_retry_times = 2@@ -223,7 +213,7 @@ class MaxRetryTimesTest(unittest.TestCase):req2 = Request(self.invalid_url)settings = {"RETRY_TIMES": middleware_max_retry_times}- spider, middleware = yield self.get_spider_and_middleware(settings)+ spider, middleware = self.get_spider_and_middleware(settings)self._test_retry(req1,@@ -240,7 +230,6 @@ class MaxRetryTimesTest(unittest.TestCase):middleware=middleware,)- @inlineCallbacksdef test_with_metakey_lesser(self):meta_max_retry_times = 4middleware_max_retry_times = 5@@ -249,7 +238,7 @@ class MaxRetryTimesTest(unittest.TestCase):req2 = Request(self.invalid_url)settings = {"RETRY_TIMES": middleware_max_retry_times}- spider, middleware = yield self.get_spider_and_middleware(settings)+ spider, middleware = self.get_spider_and_middleware(settings)self._test_retry(req1,@@ -266,10 +255,9 @@ class MaxRetryTimesTest(unittest.TestCase):middleware=middleware,)- @inlineCallbacksdef test_with_dont_retry(self):max_retry_times = 4- spider, middleware = yield self.get_spider_and_middleware()+ spider, middleware = self.get_spider_and_middleware()meta = {"max_retry_times": max_retry_times,"dont_retry": True,@@ -304,16 +292,13 @@ class MaxRetryTimesTest(unittest.TestCase):class GetRetryRequestTest(unittest.TestCase):- @inlineCallbacksdef get_spider(self, settings=None):- crawler = get_crawler(NoRequestsSpider, settings or {})- yield crawler.crawl()- return crawler.spider+ crawler = get_crawler(Spider, settings or {})+ return crawler._create_spider("foo")- @inlineCallbacksdef test_basic_usage(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()with LogCapture() as log:new_request = get_retry_request(request,@@ -337,10 +322,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_max_retries_reached(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()max_retry_times = 0with LogCapture() as log:new_request = get_retry_request(@@ -361,10 +345,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_one_retry(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()with LogCapture() as log:new_request = get_retry_request(request,@@ -389,9 +372,8 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_two_retries(self):- spider = yield self.get_spider()+ spider = self.get_spider()request = Request("https://example.com")new_request = requestmax_retry_times = 2@@ -445,10 +427,9 @@ class GetRetryRequestTest(unittest.TestCase):with self.assertRaises(TypeError):get_retry_request(request) # pylint: disable=missing-kwoa- @inlineCallbacksdef test_max_retry_times_setting(self):max_retry_times = 0- spider = yield self.get_spider({"RETRY_TIMES": max_retry_times})+ spider = self.get_spider({"RETRY_TIMES": max_retry_times})request = Request("https://example.com")new_request = get_retry_request(request,@@ -456,10 +437,9 @@ class GetRetryRequestTest(unittest.TestCase):)self.assertEqual(new_request, None)- @inlineCallbacksdef test_max_retry_times_meta(self):max_retry_times = 0- spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1})+ spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1})meta = {"max_retry_times": max_retry_times}request = Request("https://example.com", meta=meta)new_request = get_retry_request(@@ -468,10 +448,9 @@ class GetRetryRequestTest(unittest.TestCase):)self.assertEqual(new_request, None)- @inlineCallbacksdef test_max_retry_times_argument(self):max_retry_times = 0- spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1})+ spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1})meta = {"max_retry_times": max_retry_times + 1}request = Request("https://example.com", meta=meta)new_request = get_retry_request(@@ -481,10 +460,9 @@ class GetRetryRequestTest(unittest.TestCase):)self.assertEqual(new_request, None)- @inlineCallbacksdef test_priority_adjust_setting(self):priority_adjust = 1- spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust})+ spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust})request = Request("https://example.com")new_request = get_retry_request(request,@@ -492,10 +470,9 @@ class GetRetryRequestTest(unittest.TestCase):)self.assertEqual(new_request.priority, priority_adjust)- @inlineCallbacksdef test_priority_adjust_argument(self):priority_adjust = 1- spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1})+ spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1})request = Request("https://example.com")new_request = get_retry_request(request,@@ -504,10 +481,9 @@ class GetRetryRequestTest(unittest.TestCase):)self.assertEqual(new_request.priority, priority_adjust)- @inlineCallbacksdef test_log_extra_retry_success(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()with LogCapture(attributes=("spider",)) as log:get_retry_request(request,@@ -515,10 +491,9 @@ class GetRetryRequestTest(unittest.TestCase):)log.check_present(spider)- @inlineCallbacksdef test_log_extra_retries_exceeded(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()with LogCapture(attributes=("spider",)) as log:get_retry_request(request,@@ -527,10 +502,9 @@ class GetRetryRequestTest(unittest.TestCase):)log.check_present(spider)- @inlineCallbacksdef test_reason_string(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = "because"with LogCapture() as log:get_retry_request(@@ -550,10 +524,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_reason_builtin_exception(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = NotImplementedError()expected_reason_string = "builtins.NotImplementedError"with LogCapture() as log:@@ -576,10 +549,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_reason_builtin_exception_class(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = NotImplementedErrorexpected_reason_string = "builtins.NotImplementedError"with LogCapture() as log:@@ -602,10 +574,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_reason_custom_exception(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = IgnoreRequest()expected_reason_string = "scrapy.exceptions.IgnoreRequest"with LogCapture() as log:@@ -628,10 +599,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_reason_custom_exception_class(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = IgnoreRequestexpected_reason_string = "scrapy.exceptions.IgnoreRequest"with LogCapture() as log:@@ -654,11 +624,10 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_custom_logger(self):logger = logging.getLogger("custom-logger")request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = "because"with LogCapture() as log:get_retry_request(@@ -675,10 +644,9 @@ class GetRetryRequestTest(unittest.TestCase):))- @inlineCallbacksdef test_custom_stats_key(self):request = Request("https://example.com")- spider = yield self.get_spider()+ spider = self.get_spider()expected_reason = "because"stats_key = "custom_retry"get_retry_request(@@ -692,3 +660,7 @@ class GetRetryRequestTest(unittest.TestCase):f"{stats_key}/reason_count/{expected_reason}",):self.assertEqual(spider.crawler.stats.get_value(stat), 1)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 55ae0c2b7..39dfe9ab5 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -1,29 +1,27 @@import warningsfrom itertools import product--from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittest+from unittest import TestCasefrom scrapy.downloadermiddlewares.stats import DownloaderStatsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Response+from scrapy.spiders import Spiderfrom scrapy.utils.response import response_httpreprfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpiderclass MyException(Exception):pass-class TestDownloaderStats(unittest.TestCase):- @inlineCallbacks+class TestDownloaderStats(TestCase):def setUp(self):- self.crawler = get_crawler(NoRequestsSpider)- yield self.crawler.crawl()- self.spider = self.crawler.spider+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider("scrapytest.org")self.mw = DownloaderStats(self.crawler.stats)+ self.crawler.stats.open_spider(self.spider)+self.req = Request("http://scrapytest.org")self.res = Response("scrapytest.org", status=400)diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 1ba6125b2..aa0975555 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -2,25 +2,22 @@ import hashlibimport shutilimport sysimport tempfile+import unittestfrom pathlib import Pathfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittestfrom scrapy.core.scheduler import Schedulerfrom scrapy.dupefilters import RFPDupeFilterfrom scrapy.http import Requestfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpider+from tests.spiders import SimpleSpider-@inlineCallbacksdef _get_dupefilter(*, crawler=None, settings=None, open=True):if crawler is None:- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)dupefilter = scheduler.dfif open:@@ -47,51 +44,41 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter):class DirectDupeFilter:method = "n/a"- def open(self):- pass-class RFPDupeFilterTest(unittest.TestCase):- @inlineCallbacksdef test_df_from_crawler_scheduler(self):settings = {"DUPEFILTER_DEBUG": True,"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertTrue(scheduler.df.debug)self.assertEqual(scheduler.df.method, "from_crawler")- @inlineCallbacksdef test_df_from_settings_scheduler(self):settings = {"DUPEFILTER_DEBUG": True,"DUPEFILTER_CLASS": FromSettingsRFPDupeFilter,"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertTrue(scheduler.df.debug)self.assertEqual(scheduler.df.method, "from_settings")- @inlineCallbacksdef test_df_direct_scheduler(self):settings = {"DUPEFILTER_CLASS": DirectDupeFilter,"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertEqual(scheduler.df.method, "n/a")- @inlineCallbacksdef test_filter(self):- dupefilter = yield _get_dupefilter()+ dupefilter = _get_dupefilter()r1 = Request("http://scrapytest.org/1")r2 = Request("http://scrapytest.org/2")r3 = Request("http://scrapytest.org/2")@@ -104,14 +91,13 @@ class RFPDupeFilterTest(unittest.TestCase):dupefilter.close("finished")- @inlineCallbacksdef test_dupefilter_path(self):r1 = Request("http://scrapytest.org/1")r2 = Request("http://scrapytest.org/2")path = tempfile.mkdtemp()try:- df = yield _get_dupefilter(settings={"JOBDIR": path}, open=False)+ df = _get_dupefilter(settings={"JOBDIR": path}, open=False)try:df.open()assert not df.request_seen(r1)@@ -119,7 +105,7 @@ class RFPDupeFilterTest(unittest.TestCase):finally:df.close("finished")- df2 = yield _get_dupefilter(settings={"JOBDIR": path}, open=False)+ df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False)assert df != df2try:df2.open()@@ -131,13 +117,12 @@ class RFPDupeFilterTest(unittest.TestCase):finally:shutil.rmtree(path)- @inlineCallbacksdef test_request_fingerprint(self):"""Test if customization of request_fingerprint method will changeoutput of request_seen."""- dupefilter = yield _get_dupefilter()+ dupefilter = _get_dupefilter()r1 = Request("http://scrapytest.org/index.html")r2 = Request("http://scrapytest.org/INDEX.html")@@ -153,14 +138,13 @@ class RFPDupeFilterTest(unittest.TestCase):return fp.digest()settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter}- case_insensitive_dupefilter = yield _get_dupefilter(settings=settings)+ case_insensitive_dupefilter = _get_dupefilter(settings=settings)assert not case_insensitive_dupefilter.request_seen(r1)assert case_insensitive_dupefilter.request_seen(r2)case_insensitive_dupefilter.close("finished")- @inlineCallbacksdef test_seenreq_newlines(self):"""Checks against adding duplicate \r toline endings on Windows platforms."""@@ -168,8 +152,7 @@ class RFPDupeFilterTest(unittest.TestCase):r1 = Request("http://scrapytest.org/1")path = tempfile.mkdtemp()- crawler = get_crawler(NoRequestsSpider, settings_dict={"JOBDIR": path})- yield crawler.crawl()+ crawler = get_crawler(settings_dict={"JOBDIR": path})try:scheduler = Scheduler.from_crawler(crawler)df = scheduler.df@@ -178,10 +161,7 @@ class RFPDupeFilterTest(unittest.TestCase):df.close("finished")with Path(path, "requests.seen").open("rb") as seen_file:- try:- line = next(seen_file).decode()- except StopIteration:- return+ line = next(seen_file).decode()assert not line.endswith("\r\r\n")if sys.platform == "win32":assert line.endswith("\r\n")@@ -191,7 +171,6 @@ class RFPDupeFilterTest(unittest.TestCase):finally:shutil.rmtree(path)- @inlineCallbacksdef test_log(self):with LogCapture() as log:settings = {@@ -199,10 +178,9 @@ class RFPDupeFilterTest(unittest.TestCase):"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()- spider = crawler.spider- dupefilter = yield _get_dupefilter(crawler=crawler)+ crawler = get_crawler(SimpleSpider, settings_dict=settings)+ spider = SimpleSpider.from_crawler(crawler)+ dupefilter = _get_dupefilter(crawler=crawler)r1 = Request("http://scrapytest.org/index.html")r2 = Request("http://scrapytest.org/index.html")@@ -222,7 +200,6 @@ class RFPDupeFilterTest(unittest.TestCase):dupefilter.close("finished")- @inlineCallbacksdef test_log_debug(self):with LogCapture() as log:settings = {@@ -230,10 +207,9 @@ class RFPDupeFilterTest(unittest.TestCase):"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()- spider = crawler.spider- dupefilter = yield _get_dupefilter(crawler=crawler)+ crawler = get_crawler(SimpleSpider, settings_dict=settings)+ spider = SimpleSpider.from_crawler(crawler)+ dupefilter = _get_dupefilter(crawler=crawler)r1 = Request("http://scrapytest.org/index.html")r2 = Request(@@ -263,17 +239,15 @@ class RFPDupeFilterTest(unittest.TestCase):dupefilter.close("finished")- @inlineCallbacksdef test_log_debug_default_dupefilter(self):with LogCapture() as log:settings = {"DUPEFILTER_DEBUG": True,"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()- spider = crawler.spider- dupefilter = yield _get_dupefilter(crawler=crawler)+ crawler = get_crawler(SimpleSpider, settings_dict=settings)+ spider = SimpleSpider.from_crawler(crawler)+ dupefilter = _get_dupefilter(crawler=crawler)r1 = Request("http://scrapytest.org/index.html")r2 = Request(diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 5deae5146..8d7afb6a1 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -420,16 +420,12 @@ class EngineTest(unittest.TestCase):@defer.inlineCallbacksdef test_close_downloader(self):- crawler = get_crawler(TestSpider)- yield crawler.crawl()- e = ExecutionEngine(crawler, lambda _: None)+ e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)yield e.close()@defer.inlineCallbacksdef test_start_already_running_exception(self):- crawler = get_crawler(TestSpider)- yield crawler.crawl()- e = ExecutionEngine(crawler, lambda _: None)+ e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)yield e.open_spider(TestSpider(), [])e.start()try:diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex ce8452b5f..6e5fb0325 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -1,8 +1,6 @@import datetimeimport typing--from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittest+import unittestfrom scrapy.crawler import Crawlerfrom scrapy.extensions.periodic_log import PeriodicLog@@ -61,44 +59,33 @@ class TestExtPeriodicLog(PeriodicLog):self.stats._stats = stats_dump_2-@inlineCallbacksdef extension(settings=None):- crawler = Crawler(- MetaSpider,- settings=settings,- )- yield crawler.crawl()+ crawler = Crawler(MetaSpider, settings=settings)+ crawler._load_settings()return TestExtPeriodicLog.from_crawler(crawler)class TestPeriodicLog(unittest.TestCase):- @inlineCallbacksdef test_extension_enabled(self):# Expected that settings for this extension loaded succesfully# And on certain conditions - extension raising NotConfigured# "PERIODIC_LOG_STATS": True -> set to {"enabled": True}# due to TypeError exception from settings.getdict- assert (yield extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}))+ assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})# "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}# due to JSONDecodeError(ValueError) exception from settings.getdict- assert (- yield extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})- )+ assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})# The ame for PERIODIC_LOG_DELTA:- assert (yield extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}))- assert (- yield extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})- )+ assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})+ assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})- @inlineCallbacksdef test_log_delta(self):- @inlineCallbacksdef emulate(settings=None):spider = MetaSpider()- ext = yield extension(settings)+ ext = extension(settings)ext.spider_opened(spider)ext.set_a()a = ext.log_delta()@@ -107,9 +94,8 @@ class TestPeriodicLog(unittest.TestCase):ext.spider_closed(spider, reason="finished")return ext, a, b- @inlineCallbacksdef check(settings: dict, condition: typing.Callable):- ext, a, b = yield emulate(settings)+ ext, a, b = emulate(settings)assert list(a["delta"].keys()) == [k for k, v in ext.stats._stats.items() if condition(k, v)]@@ -118,49 +104,45 @@ class TestPeriodicLog(unittest.TestCase):]# Including all- yield check(- {"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))- )+ check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))# include:- yield check(+ check({"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,)# include multiple- yield check(+ check({"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},lambda k, v: isinstance(v, (int, float))and ("downloader/" in k or "scheduler/" in k),)# exclude- yield check(+ check({"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,)# exclude multiple- yield check(+ check({"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},lambda k, v: isinstance(v, (int, float))and ("downloader/" not in k and "scheduler/" not in k),)# include exclude combined- yield check(+ check({"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},lambda k, v: isinstance(v, (int, float))and ("downloader/" in k and "bytes" not in k),)- @inlineCallbacksdef test_log_stats(self):- @inlineCallbacksdef emulate(settings=None):spider = MetaSpider()- ext = yield extension(settings)+ ext = extension(settings)ext.spider_opened(spider)ext.set_a()a = ext.log_crawler_stats()@@ -169,9 +151,8 @@ class TestPeriodicLog(unittest.TestCase):ext.spider_closed(spider, reason="finished")return ext, a, b- @inlineCallbacksdef check(settings: dict, condition: typing.Callable):- ext, a, b = yield emulate(settings)+ ext, a, b = emulate(settings)assert list(a["stats"].keys()) == [k for k, v in ext.stats._stats.items() if condition(k, v)]@@ -180,34 +161,35 @@ class TestPeriodicLog(unittest.TestCase):]# Including all- yield check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)+ check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)# include:- yield check(+ check({"PERIODIC_LOG_STATS": {"include": ["downloader/"]}},lambda k, v: "downloader/" in k,)# include multiple- yield check(+ check({"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}},lambda k, v: "downloader/" in k or "scheduler/" in k,)# exclude- yield check(+ check({"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}},lambda k, v: "downloader/" not in k,)# exclude multiple- yield check(+ check({"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}},lambda k, v: "downloader/" not in k and "scheduler/" not in k,)# include exclude combined- yield check(+ check({"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},lambda k, v: "downloader/" in k and "bytes" not in k,)+ #diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex 6f35510b7..9fd680e9f 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -9,7 +9,7 @@ from scrapy.utils.test import get_crawlerclass TelnetExtensionTest(unittest.TestCase):def _get_console_and_portal(self, settings=None):- crawler = get_crawler(settings_dict=settings, disable_telnet=False)+ crawler = get_crawler(settings_dict=settings)console = TelnetConsole(crawler)# This function has some side effects we don't need for this testdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 875abdb1f..6b82974fa 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -26,7 +26,6 @@ import lxml.etreeimport pytestfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom twisted.trial import unittestfrom w3lib.url import file_uri_to_path, path_to_file_urifrom zope.interface import implementer@@ -52,7 +51,7 @@ from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_botofrom tests.mockserver import MockFTPServer, MockServer-from tests.spiders import ItemSpider, NoRequestsSpider+from tests.spiders import ItemSpiderdef path_to_url(path):@@ -2748,14 +2747,13 @@ class FeedExporterSignalsTest(unittest.TestCase):d.callback(None)return d- @inlineCallbacksdef run_signaled_feed_exporter(self, feed_exporter_signal_handler, feed_slot_signal_handler):- crawler = get_crawler(NoRequestsSpider, settings_dict=self.settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=self.settings)feed_exporter = FeedExporter.from_crawler(crawler)- spider = crawler.spider+ spider = scrapy.Spider("default")+ spider.crawler = crawlercrawler.signals.connect(feed_exporter_signal_handler,signal=signals.feed_exporter_closed,@@ -2766,26 +2764,24 @@ class FeedExporterSignalsTest(unittest.TestCase):feed_exporter.open_spider(spider)for item in self.items:feed_exporter.item_scraped(item, spider)- yield defer.ensureDeferred(feed_exporter.close_spider(spider))+ defer.ensureDeferred(feed_exporter.close_spider(spider))- @inlineCallbacksdef test_feed_exporter_signals_sent(self):self.feed_exporter_closed_received = Falseself.feed_slot_closed_received = False- yield self.run_signaled_feed_exporter(+ self.run_signaled_feed_exporter(self.feed_exporter_closed_signal_handler,self.feed_slot_closed_signal_handler,)self.assertTrue(self.feed_slot_closed_received)self.assertTrue(self.feed_exporter_closed_received)- @inlineCallbacksdef test_feed_exporter_signals_sent_deferred(self):self.feed_exporter_closed_received = Falseself.feed_slot_closed_received = False- yield self.run_signaled_feed_exporter(+ self.run_signaled_feed_exporter(self.feed_exporter_closed_signal_handler_deferred,self.feed_slot_closed_signal_handler_deferred,)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 0d769f29b..bf96f17b6 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -15,7 +15,6 @@ import attrimport pytestfrom itemadapter import ItemAdapterfrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacksfrom twisted.trial import unittestfrom scrapy.http import Request, Response@@ -36,18 +35,15 @@ from scrapy.utils.test import (skip_if_no_boto,)from tests.mockserver import MockFTPServer-from tests.spiders import NoRequestsSpiderfrom .test_pipeline_media import _mocked_download_funcclass FilesPipelineTestCase(unittest.TestCase):- @inlineCallbacksdef setUp(self):self.tempdir = mkdtemp()settings_dict = {"FILES_STORE": self.tempdir}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)self.pipeline = FilesPipeline.from_crawler(crawler)self.pipeline.download_func = _mocked_download_funcself.pipeline.open_spider(None)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 8a13ea552..d655eb128 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -14,11 +14,11 @@ from scrapy.pipelines.files import FileExceptionfrom scrapy.pipelines.images import ImagesPipelinefrom scrapy.pipelines.media import MediaPipelinefrom scrapy.settings import Settings+from scrapy.spiders import Spiderfrom scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_allfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpidertry:from PIL import Image # noqa: imported just to check for the import error@@ -40,11 +40,10 @@ class BaseMediaPipelineTestCase(unittest.TestCase):pipeline_class = MediaPipelinesettings = None- @inlineCallbacksdef setUp(self):- crawler = get_crawler(NoRequestsSpider, self.settings)- yield crawler.crawl()- self.spider = crawler.spider+ spider_cls = Spider+ self.spider = spider_cls("media.com")+ crawler = get_crawler(spider_cls, self.settings)self.pipe = self.pipeline_class.from_crawler(crawler)self.pipe.download_func = _mocked_download_funcself.pipe.open_spider(self.spider)@@ -431,14 +430,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):skip = skip_pillow- @inlineCallbacksdef setUp(self):settings_dict = {"IMAGES_STORE": "store-uri","IMAGES_THUMBS": {"small": (50, 50)},}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)- yield crawler.crawl()+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)self.pipe.download_func = _mocked_download_funcself.pipe.open_spider(None)diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 54c77eb68..ef9b360c4 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -1,10 +1,10 @@import collectionsimport shutilimport tempfile+import unittestfrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittest+from twisted.trial.unittest import TestCasefrom scrapy.core.downloader import Downloaderfrom scrapy.core.scheduler import Scheduler@@ -15,7 +15,6 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_objectfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer-from tests.spiders import NoRequestsSpiderMockEngine = collections.namedtuple("MockEngine", ["downloader"])MockSlot = collections.namedtuple("MockSlot", ["active"])@@ -310,7 +309,7 @@ class StartUrlsSpider(Spider):pass-class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase):+class TestIntegrationWithDownloaderAwareInMemory(TestCase):def setUp(self):self.crawler = get_crawler(spidercls=StartUrlsSpider,@@ -337,19 +336,16 @@ class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase):class TestIncompatibility(unittest.TestCase):- @inlineCallbacksdef _incompatible(self):settings = dict(SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue",CONCURRENT_REQUESTS_PER_IP=1,)- crawler = get_crawler(NoRequestsSpider, settings)- yield crawler.crawl()- spider = crawler.spider+ crawler = get_crawler(Spider, settings)scheduler = Scheduler.from_crawler(crawler)+ spider = Spider(name="spider")scheduler.open(spider)- @inlineCallbacksdef test_incompatibility(self):with self.assertRaises(ValueError):- yield self._incompatible()+ self._incompatible()diff --git a/tests/test_spider.py b/tests/test_spider.pyindex a88d9b505..00da3d485 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -114,7 +114,7 @@ class SpiderTest(unittest.TestCase):spider.settings.set("TEST1", "spider_instance", priority="spider")return spider- crawler = get_crawler(TestSpider, settings_dict=project_settings)+ crawler = Crawler(TestSpider, project_settings)self.assertEqual(crawler.settings.get("TEST1"), "spider")self.assertEqual(crawler.settings.get("TEST2"), "spider")self.assertEqual(crawler.settings.get("TEST3"), "project")diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 2fcb5b364..1d5a887cc 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -1,9 +1,9 @@import logging+from unittest import TestCasefrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.internet.defer import inlineCallbacks-from twisted.trial.unittest import TestCase+from twisted.trial.unittest import TestCase as TrialTestCasefrom scrapy.http import Request, Responsefrom scrapy.settings import Settings@@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddlewarefrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer-from tests.spiders import MockServerSpider, NoRequestsSpider+from tests.spiders import MockServerSpiderclass _HttpErrorSpider(MockServerSpider):@@ -59,11 +59,9 @@ def _responses(request, status_codes):class TestHttpErrorMiddleware(TestCase):- @inlineCallbacksdef setUp(self):- crawler = get_crawler(NoRequestsSpider)- yield crawler.crawl()- self.spider = crawler.spider+ crawler = get_crawler(Spider)+ self.spider = Spider.from_crawler(crawler, name="foo")self.mw = HttpErrorMiddleware(Settings({}))self.req = Request("http://scrapytest.org")self.res200, self.res404 = _responses(self.req, [200, 404])@@ -173,7 +171,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):self.assertIsNone(mw.process_spider_input(res402, self.spider))-class TestHttpErrorMiddlewareIntegrational(TestCase):+class TestHttpErrorMiddlewareIntegrational(TrialTestCase):def setUp(self):self.mockserver = MockServer()self.mockserver.__enter__()diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex ed00c0a6b..ea45b7698 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -1,21 +1,17 @@import warnings+from unittest import TestCasefrom urllib.parse import urlparse-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittest-from scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarningfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-class TestOffsiteMiddleware(unittest.TestCase):- @inlineCallbacks+class TestOffsiteMiddleware(TestCase):def setUp(self):crawler = get_crawler(Spider)- yield crawler.crawl(**self._get_spiderargs())- self.spider = crawler.spider+ self.spider = crawler._create_spider(**self._get_spiderargs())self.mw = OffsiteMiddleware.from_crawler(crawler)self.mw.spider_opened(self.spider)diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex a3c284484..9111e4c82 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -1,23 +1,21 @@+from unittest import TestCase+from testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittestfrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.spidermiddlewares.urllength import UrlLengthMiddleware+from scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpider-class TestUrlLengthMiddleware(unittest.TestCase):- @inlineCallbacks+class TestUrlLengthMiddleware(TestCase):def setUp(self):self.maxlength = 25settings = Settings({"URLLENGTH_LIMIT": self.maxlength})- crawler = get_crawler(NoRequestsSpider)- yield crawler.crawl()- self.spider = crawler.spider+ crawler = get_crawler(Spider)+ self.spider = crawler._create_spider("foo")self.stats = crawler.statsself.mw = UrlLengthMiddleware.from_settings(settings)diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 2117e65b1..eae744df5 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -1,10 +1,9 @@import loggingimport sys+import unittestfrom testfixtures import LogCapture-from twisted.internet.defer import inlineCallbacksfrom twisted.python.failure import Failure-from twisted.trial import unittestfrom scrapy.extensions import telnetfrom scrapy.utils.log import (@@ -14,7 +13,6 @@ from scrapy.utils.log import (failure_to_exc_info,)from scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpiderclass FailureToExcInfoTest(unittest.TestCase):@@ -62,7 +60,6 @@ class TopLevelFormatterTest(unittest.TestCase):class LogCounterHandlerTest(unittest.TestCase):- @inlineCallbacksdef setUp(self):settings = {"LOG_LEVEL": "WARNING"}if not telnet.TWISTED_CONCH_AVAILABLE:@@ -71,8 +68,7 @@ class LogCounterHandlerTest(unittest.TestCase):self.logger = logging.getLogger("test")self.logger.setLevel(logging.NOTSET)self.logger.propagate = False- self.crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield self.crawler.crawl()+ self.crawler = get_crawler(settings_dict=settings)self.handler = LogCounterHandler(self.crawler)self.logger.addHandler(self.handler)diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 9ca9faa0c..e6d1abe3f 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -1,12 +1,11 @@import json+import unittestimport warningsfrom hashlib import sha1from typing import Dict, Mapping, Optional, Tuple, Unionfrom weakref import WeakKeyDictionaryimport pytest-from twisted.internet.defer import inlineCallbacks-from twisted.trial import unittestfrom w3lib.url import canonicalize_urlfrom scrapy.http import Request@@ -23,7 +22,6 @@ from scrapy.utils.request import (request_to_curl,)from scrapy.utils.test import get_crawler-from tests.spiders import NoRequestsSpiderclass UtilsRequestTest(unittest.TestCase):@@ -451,18 +449,15 @@ class BackwardCompatibilityTestCase(unittest.TestCase):)self.assertEqual(fp, old_fp)- @inlineCallbacksdef test_component_backward_compatibility(self):for request_object in REQUEST_OBJECTS_TO_TEST:with warnings.catch_warnings():warnings.simplefilter("ignore")- crawler = get_crawler(NoRequestsSpider, prevent_warnings=False)- yield crawler.crawl()+ crawler = get_crawler(prevent_warnings=False)fp = crawler.request_fingerprinter.fingerprint(request_object)old_fp = request_fingerprint_2_6(request_object)self.assertEqual(fp.hex(), old_fp)- @inlineCallbacksdef test_custom_component_backward_compatibility(self):"""Tests that the backward-compatible request fingerprinting class featuredin the documentation is indeed backward compatible and does not cause a@@ -485,8 +480,7 @@ class BackwardCompatibilityTestCase(unittest.TestCase):settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)fp = crawler.request_fingerprinter.fingerprint(request_object)old_fp = request_fingerprint_2_6(request_object)self.assertEqual(fp.hex(), old_fp)@@ -494,11 +488,9 @@ class BackwardCompatibilityTestCase(unittest.TestCase):class RequestFingerprinterTestCase(unittest.TestCase):- @inlineCallbacksdef test_default_implementation(self):with warnings.catch_warnings(record=True) as logged_warnings:- crawler = get_crawler(NoRequestsSpider, prevent_warnings=False)- yield crawler.crawl()+ crawler = get_crawler(prevent_warnings=False)request = Request("https://example.com")self.assertEqual(crawler.request_fingerprinter.fingerprint(request),@@ -506,14 +498,12 @@ class RequestFingerprinterTestCase(unittest.TestCase):)self.assertTrue(logged_warnings)- @inlineCallbacksdef test_deprecated_implementation(self):settings = {"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",}with warnings.catch_warnings(record=True) as logged_warnings:- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)request = Request("https://example.com")self.assertEqual(crawler.request_fingerprinter.fingerprint(request),@@ -521,14 +511,12 @@ class RequestFingerprinterTestCase(unittest.TestCase):)self.assertTrue(logged_warnings)- @inlineCallbacksdef test_recommended_implementation(self):settings = {"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",}with warnings.catch_warnings(record=True) as logged_warnings:- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)request = Request("https://example.com")self.assertEqual(crawler.request_fingerprinter.fingerprint(request),@@ -536,18 +524,15 @@ class RequestFingerprinterTestCase(unittest.TestCase):)self.assertFalse(logged_warnings)- @inlineCallbacksdef test_unknown_implementation(self):settings = {"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",}with self.assertRaises(ValueError):- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ get_crawler(settings_dict=settings)class CustomRequestFingerprinterTestCase(unittest.TestCase):- @inlineCallbacksdef test_include_headers(self):class RequestFingerprinter:def fingerprint(self, request):@@ -556,8 +541,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)r1 = Request("http://www.example.com", headers={"X-ID": "1"})fp1 = crawler.request_fingerprinter.fingerprint(r1)@@ -565,7 +549,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):fp2 = crawler.request_fingerprinter.fingerprint(r2)self.assertNotEqual(fp1, fp2)- @inlineCallbacksdef test_dont_canonicalize(self):class RequestFingerprinter:cache = WeakKeyDictionary()@@ -580,8 +563,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)r1 = Request("http://www.example.com?a=1&a=2")fp1 = crawler.request_fingerprinter.fingerprint(r1)@@ -589,7 +571,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):fp2 = crawler.request_fingerprinter.fingerprint(r2)self.assertNotEqual(fp1, fp2)- @inlineCallbacksdef test_meta(self):class RequestFingerprinter:def fingerprint(self, request):@@ -600,8 +581,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)r1 = Request("http://www.example.com")fp1 = crawler.request_fingerprinter.fingerprint(r1)@@ -616,7 +596,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):self.assertNotEqual(fp2, fp4)self.assertEqual(fp2, fp3)- @inlineCallbacksdef test_from_crawler(self):class RequestFingerprinter:@classmethod@@ -633,14 +612,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,"FINGERPRINT": b"fingerprint",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)request = Request("http://www.example.com")fingerprint = crawler.request_fingerprinter.fingerprint(request)self.assertEqual(fingerprint, settings["FINGERPRINT"])- @inlineCallbacksdef test_from_settings(self):class RequestFingerprinter:@classmethod@@ -657,14 +634,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,"FINGERPRINT": b"fingerprint",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)request = Request("http://www.example.com")fingerprint = crawler.request_fingerprinter.fingerprint(request)self.assertEqual(fingerprint, settings["FINGERPRINT"])- @inlineCallbacksdef test_from_crawler_and_settings(self):class RequestFingerprinter:# This method is ignored due to the presence of from_crawler@@ -686,8 +661,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,"FINGERPRINT": b"fingerprint",}- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)- yield crawler.crawl()+ crawler = get_crawler(settings_dict=settings)request = Request("http://www.example.com")fingerprint = crawler.request_fingerprinter.fingerprint(request)@@ -754,3 +728,7 @@ class RequestToCurlTest(unittest.TestCase):" --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'")self._test_request(request_object, expected_curl_command)+++if __name__ == "__main__":+ unittest.main()