Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add Crawler._load_settings().

Andrey Rakhmatullinbb15c93a2bbd
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 0a5e61f7a..71f43365d 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -77,6 +77,7 @@ class Command(ScrapyCommand):
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
crawler = self.crawler_process._create_crawler(spidercls)
+ crawler._load_settings()
# The Shell class needs a persistent engine in the crawler
crawler.engine = crawler._create_engine()
crawler.engine.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 49034c9f1..67e44541d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -82,6 +82,7 @@ class Crawler:
self._init_reactor: bool = init_reactor
self.crawling: bool = False
+ self._settings_loaded: bool = False
self._started: bool = False
self.extensions: Optional[ExtensionManager] = None
self.stats: Optional[StatsCollector] = None
@@ -90,6 +91,53 @@ class Crawler:
self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
+ def _load_settings(self) -> None:
+ if self._settings_loaded:
+ return
+ self._settings_loaded = True
+
+ self.addons.load_settings(self.settings)
+ self.stats = load_object(self.settings["STATS_CLASS"])(self)
+
+ handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
+ logging.root.addHandler(handler)
+ # lambda is assigned to Crawler attribute because this way it is not
+ # garbage collected after leaving the scope
+ self.__remove_handler = lambda: logging.root.removeHandler(handler)
+ self.signals.connect(self.__remove_handler, signals.engine_stopped)
+
+ lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
+ self.logformatter = lf_cls.from_crawler(self)
+
+ self.request_fingerprinter = create_instance(
+ load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
+ settings=self.settings,
+ crawler=self,
+ )
+
+ reactor_class: str = self.settings["TWISTED_REACTOR"]
+ event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
+ if self._init_reactor:
+ # this needs to be done after the spider settings are merged,
+ # but before something imports twisted.internet.reactor
+ if reactor_class:
+ install_reactor(reactor_class, event_loop)
+ else:
+ from twisted.internet import reactor # noqa: F401
+ log_reactor_info()
+ if reactor_class:
+ verify_installed_reactor(reactor_class)
+ if is_asyncio_reactor_installed() and event_loop:
+ verify_installed_asyncio_event_loop(event_loop)
+
+ self.extensions = ExtensionManager.from_crawler(self)
+ self.settings.freeze()
+
+ d = dict(overridden_settings(self.settings))
+ logger.info(
+ "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
+ )
+
@inlineCallbacks
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
if self.crawling:
@@ -104,49 +152,7 @@ class Crawler:
try:
self.spider = self._create_spider(*args, **kwargs)
-
- self.addons.load_settings(self.settings)
- self.stats = load_object(self.settings["STATS_CLASS"])(self)
-
- handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
- logging.root.addHandler(handler)
- # lambda is assigned to Crawler attribute because this way it is not
- # garbage collected after leaving the scope
- self.__remove_handler = lambda: logging.root.removeHandler(handler)
- self.signals.connect(self.__remove_handler, signals.engine_stopped)
-
- lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
- self.logformatter = lf_cls.from_crawler(self)
-
- self.request_fingerprinter = create_instance(
- load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
- settings=self.settings,
- crawler=self,
- )
-
- reactor_class: str = self.settings["TWISTED_REACTOR"]
- event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
- if self._init_reactor:
- # this needs to be done after the spider settings are merged,
- # but before something imports twisted.internet.reactor
- if reactor_class:
- install_reactor(reactor_class, event_loop)
- else:
- from twisted.internet import reactor # noqa: F401
- log_reactor_info()
- if reactor_class:
- verify_installed_reactor(reactor_class)
- if is_asyncio_reactor_installed() and event_loop:
- verify_installed_asyncio_event_loop(event_loop)
-
- self.extensions = ExtensionManager.from_crawler(self)
- self.settings.freeze()
-
- d = dict(overridden_settings(self.settings))
- logger.info(
- "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
- )
-
+ self._load_settings()
self.engine = self._create_engine()
start_requests = iter(self.spider.start_requests())
yield self.engine.open_spider(self.spider, start_requests)
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 44a30dc15..9397e78b9 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -69,28 +69,30 @@ def get_ftp_content_and_delete(
return b"".join(ftp_data)
+class TestSpider(Spider):
+ name = "test"
+
+
def get_crawler(
spidercls: Optional[Type[Spider]] = None,
settings_dict: Optional[Dict[str, Any]] = None,
prevent_warnings: bool = True,
- disable_telnet: bool = True,
) -> Crawler:
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
priority.
"""
from scrapy.crawler import CrawlerRunner
- from scrapy.spiders import Spider
# Set by default settings that prevent deprecation warnings.
settings: Dict[str, Any] = {}
if prevent_warnings:
settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7"
- if disable_telnet:
- settings["TELNETCONSOLE_ENABLED"] = False
settings.update(settings_dict or {})
runner = CrawlerRunner(settings)
- return runner.create_crawler(spidercls or Spider)
+ crawler = runner.create_crawler(spidercls or TestSpider)
+ crawler._load_settings()
+ return crawler
def get_pythonpath() -> str:
diff --git a/tests/spiders.py b/tests/spiders.py
index eeb0194eb..f29dea2a1 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -18,13 +18,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
-class NoRequestsSpider(Spider):
- name = "no_request"
-
- def start_requests(self):
- return []
-
-
class MockServerSpider(Spider):
def __init__(self, mockserver=None, *args, **kwargs):
super().__init__(*args, **kwargs)
diff --git a/tests/test_addons.py b/tests/test_addons.py
index c4b4b7ac9..aa1b760c2 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -1,15 +1,13 @@
import itertools
+import unittest
from typing import Any, Dict
from unittest.mock import patch
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
-
+from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerRunner
from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
class SimpleAddon:
@@ -53,17 +51,14 @@ class AddonTest(unittest.TestCase):
class AddonManagerTest(unittest.TestCase):
- @inlineCallbacks
def test_load_settings(self):
settings_dict = {
"ADDONS": {"tests.test_addons.SimpleAddon": 0},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
manager = crawler.addons
self.assertIsInstance(manager.addons[0], SimpleAddon)
- @inlineCallbacks
def test_notconfigured(self):
class NotConfiguredAddon:
def update_settings(self, settings):
@@ -72,12 +67,10 @@ class AddonManagerTest(unittest.TestCase):
settings_dict = {
"ADDONS": {NotConfiguredAddon: 0},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
manager = crawler.addons
self.assertFalse(manager.addons)
- @inlineCallbacks
def test_load_settings_order(self):
# Get three addons with different settings
addonlist = []
@@ -89,25 +82,21 @@ class AddonManagerTest(unittest.TestCase):
for ordered_addons in itertools.permutations(addonlist):
expected_order = [a.number for a in ordered_addons]
settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
manager = crawler.addons
self.assertEqual([a.number for a in manager.addons], expected_order)
self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1])
- @inlineCallbacks
def test_create_instance(self):
settings_dict = {
"ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},
"MYADDON": {"MYADDON_KEY": "val"},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
manager = crawler.addons
self.assertIsInstance(manager.addons[0], CreateInstanceAddon)
self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")
- @inlineCallbacks
def test_settings_priority(self):
config = {
"KEY": 15, # priority=addon
@@ -115,15 +104,14 @@ class AddonManagerTest(unittest.TestCase):
settings_dict = {
"ADDONS": {get_addon_cls(config): 1},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
self.assertEqual(crawler.settings.getint("KEY"), 15)
settings = Settings(settings_dict)
settings.set("KEY", 0, priority="default")
runner = CrawlerRunner(settings)
- crawler = runner.create_crawler(NoRequestsSpider)
- yield crawler.crawl()
+ crawler = runner.create_crawler(Spider)
+ crawler._load_settings()
self.assertEqual(crawler.settings.getint("KEY"), 15)
settings_dict = {
@@ -133,11 +121,9 @@ class AddonManagerTest(unittest.TestCase):
settings = Settings(settings_dict)
settings.set("KEY", 0, priority="default")
runner = CrawlerRunner(settings)
- crawler = runner.create_crawler(NoRequestsSpider)
- yield crawler.crawl()
+ crawler = runner.create_crawler(Spider)
self.assertEqual(crawler.settings.getint("KEY"), 20)
- @inlineCallbacks
def test_fallback_workflow(self):
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
@@ -154,8 +140,7 @@ class AddonManagerTest(unittest.TestCase):
settings_dict = {
"ADDONS": {AddonWithFallback: 1},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
self.assertEqual(
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler"
)
@@ -168,14 +153,12 @@ class AddonManagerTest(unittest.TestCase):
"ADDONS": {AddonWithFallback: 1},
"DOWNLOAD_HANDLERS": {"https": "UserHandler"},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
self.assertEqual(
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler"
)
self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler")
- @inlineCallbacks
def test_logging_message(self):
class LoggedAddon:
def update_settings(self, settings):
@@ -188,8 +171,7 @@ class AddonManagerTest(unittest.TestCase):
}
addon = LoggedAddon()
create_instance_mock.return_value = addon
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings_dict)
logger_mock.info.assert_called_once_with(
"Enabled addons:\n%(addons)s",
{"addons": [addon]},
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 72d06deab..6589381f3 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -7,8 +7,6 @@ from scrapy.utils.testproc import ProcessTest
from scrapy.utils.testsite import SiteTest
from tests import NON_EXISTING_RESOLVABLE, tests_datadir
-raise unittest.SkipTest("Broken for now")
-
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
command = "shell"
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 05be33c73..b1d7be628 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -26,8 +26,7 @@ from scrapy.commands.startproject import IGNORE
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_testenv
-from tests.spiders import NoRequestsSpider
-from tests.test_crawler import ExceptionSpider
+from tests.test_crawler import ExceptionSpider, NoRequestsSpider
class CommandSettings(unittest.TestCase):
@@ -713,7 +712,7 @@ class BadSpider(scrapy.Spider):
def test_run_good_spider(self):
proc, _, _ = self.runspider(
- "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider)
+ "import scrapy\n" + inspect.getsource(NoRequestsSpider)
)
ret = proc.returncode
self.assertEqual(ret, 0)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f962cecc8..120991ae7 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -10,7 +10,6 @@ import pytest
from packaging.version import parse as parse_version
from pytest import mark, raises
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from w3lib import __version__ as w3lib_version
@@ -24,7 +23,6 @@ from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer, get_mockserver_env
-from tests.spiders import NoRequestsSpider
class BaseCrawlerTest(unittest.TestCase):
@@ -72,7 +70,6 @@ class CrawlerLoggingTestCase(unittest.TestCase):
get_crawler(MySpider)
assert get_scrapy_root_handler() is None
- @inlineCallbacks
def test_spider_custom_settings_log_level(self):
log_file = Path(self.mktemp())
log_file.write_text("previous message\n", encoding="utf-8")
@@ -82,20 +79,20 @@ class CrawlerLoggingTestCase(unittest.TestCase):
custom_settings = {
"LOG_LEVEL": "INFO",
"LOG_FILE": str(log_file),
+ # settings to avoid extra warnings
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+ "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
}
- def start_requests(self):
- logging.debug("debug message")
- logging.info("info message")
- logging.warning("warning message")
- logging.error("error message")
- return []
-
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
crawler = get_crawler(MySpider)
- yield crawler.crawl()
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
+ info_count = crawler.stats.get_value("log_count/INFO")
+ logging.debug("debug message")
+ logging.info("info message")
+ logging.warning("warning message")
+ logging.error("error message")
logged = log_file.read_text(encoding="utf-8")
@@ -106,7 +103,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
self.assertIn("error message", logged)
self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1)
self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1)
- # self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) TODO
+ self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1)
self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0)
def test_spider_custom_settings_log_append(self):
@@ -183,6 +180,13 @@ class ExceptionSpider(scrapy.Spider):
raise ValueError("Exception in from_crawler method")
+class NoRequestsSpider(scrapy.Spider):
+ name = "no_request"
+
+ def start_requests(self):
+ return []
+
+
@mark.usefixtures("reactor_pytest")
class CrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index d64651211..062e8a8b4 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -3,33 +3,33 @@ from unittest import mock
from pytest import mark
from twisted.internet import defer
-from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
+from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-from tests.spiders import NoRequestsSpider
class ManagerTestCase(TestCase):
settings_dict = None
- @inlineCallbacks
def setUp(self):
- self.crawler = get_crawler(NoRequestsSpider, self.settings_dict)
- yield self.crawler.crawl()
- self.spider = self.crawler.spider
+ self.crawler = get_crawler(Spider, self.settings_dict)
+ self.spider = self.crawler._create_spider("foo")
self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler)
- yield self.mwman.open_spider(self.spider)
+ # some mw depends on stats collector
+ self.crawler.stats.open_spider(self.spider)
+ return self.mwman.open_spider(self.spider)
def tearDown(self):
+ self.crawler.stats.close_spider(self.spider, "")
return self.mwman.close_spider(self.spider)
- @inlineCallbacks
def _download(self, request, response=None):
"""Executes downloader mw manager's download method and returns
the result (Request or Response) or raise exception in case of
@@ -41,21 +41,26 @@ class ManagerTestCase(TestCase):
def download_func(**kwargs):
return response
- ret = yield self.mwman.download(download_func, request, self.spider)
+ dfd = self.mwman.download(download_func, request, self.spider)
+ # catch deferred result and return the value
+ results = []
+ dfd.addBoth(results.append)
+ self._wait(dfd)
+ ret = results[0]
+ if isinstance(ret, Failure):
+ ret.raiseException()
return ret
class DefaultsTest(ManagerTestCase):
"""Tests default behavior with default settings"""
- @inlineCallbacks
def test_request_response(self):
req = Request("http://example.com/index.html")
resp = Response(req.url, status=200)
- ret = yield self._download(req, resp)
+ ret = self._download(req, resp)
self.assertTrue(isinstance(ret, Response), "Non-response returned")
- @inlineCallbacks
def test_3xx_and_invalid_gzipped_body_must_redirect(self):
"""Regression test for a failure when redirecting a compressed
request.
@@ -81,7 +86,7 @@ class DefaultsTest(ManagerTestCase):
"Location": "http://example.com/login",
},
)
- ret = yield self._download(request=req, response=resp)
+ ret = self._download(request=req, response=resp)
self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}")
self.assertEqual(
to_bytes(ret.url),
@@ -89,7 +94,6 @@ class DefaultsTest(ManagerTestCase):
"Not redirected to location header",
)
- @inlineCallbacks
def test_200_and_invalid_gzipped_body_must_fail(self):
req = Request("http://example.com")
body = b"<p>You are being redirected</p>"
@@ -104,14 +108,12 @@ class DefaultsTest(ManagerTestCase):
"Location": "http://example.com/login",
},
)
- with self.assertRaises(OSError):
- yield self._download(request=req, response=resp)
+ self.assertRaises(OSError, self._download, request=req, response=resp)
class ResponseFromProcessRequestTest(ManagerTestCase):
"""Tests middleware returning a response from process_request."""
- @inlineCallbacks
def test_download_func_not_called(self):
resp = Response("http://example.com/index.html")
@@ -123,8 +125,12 @@ class ResponseFromProcessRequestTest(ManagerTestCase):
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- result = yield self.mwman.download(download_func, req, self.spider)
- self.assertIs(result, resp)
+ dfd = self.mwman.download(download_func, req, self.spider)
+ results = []
+ dfd.addBoth(results.append)
+ self._wait(dfd)
+
+ self.assertIs(results[0], resp)
self.assertFalse(download_func.called)
@@ -191,7 +197,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
class MiddlewareUsingDeferreds(ManagerTestCase):
"""Middlewares using Deferreds should work"""
- @inlineCallbacks
def test_deferred(self):
resp = Response("http://example.com/index.html")
@@ -208,8 +213,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
self.mwman._add_middleware(DeferredMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- result = yield self.mwman.download(download_func, req, self.spider)
- self.assertIs(result, resp)
+ dfd = self.mwman.download(download_func, req, self.spider)
+ results = []
+ dfd.addBoth(results.append)
+ self._wait(dfd)
+
+ self.assertIs(results[0], resp)
self.assertFalse(download_func.called)
@@ -217,7 +226,6 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
class MiddlewareUsingCoro(ManagerTestCase):
"""Middlewares using asyncio coroutines should work"""
- @inlineCallbacks
def test_asyncdef(self):
resp = Response("http://example.com/index.html")
@@ -229,12 +237,15 @@ class MiddlewareUsingCoro(ManagerTestCase):
self.mwman._add_middleware(CoroMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- result = yield self.mwman.download(download_func, req, self.spider)
- self.assertIs(result, resp)
+ dfd = self.mwman.download(download_func, req, self.spider)
+ results = []
+ dfd.addBoth(results.append)
+ self._wait(dfd)
+
+ self.assertIs(results[0], resp)
self.assertFalse(download_func.called)
@mark.only_asyncio()
- @inlineCallbacks
def test_asyncdef_asyncio(self):
resp = Response("http://example.com/index.html")
@@ -247,6 +258,10 @@ class MiddlewareUsingCoro(ManagerTestCase):
self.mwman._add_middleware(CoroMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- result = yield self.mwman.download(download_func, req, self.spider)
- self.assertIs(result, resp)
+ dfd = self.mwman.download(download_func, req, self.spider)
+ results = []
+ dfd.addBoth(results.append)
+ self._wait(dfd)
+
+ self.assertIs(results[0], resp)
self.assertFalse(download_func.called)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index a8a687da7..f80eff3e6 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -2,31 +2,27 @@ import email.utils
import shutil
import tempfile
import time
+import unittest
from contextlib import contextmanager
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
-
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import HtmlResponse, Request, Response
from scrapy.settings import Settings
+from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
class _BaseTest(unittest.TestCase):
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
- @inlineCallbacks
def setUp(self):
self.yesterday = email.utils.formatdate(time.time() - 86400)
self.today = email.utils.formatdate()
self.tomorrow = email.utils.formatdate(time.time() + 86400)
- self.crawler = get_crawler(NoRequestsSpider)
- yield self.crawler.crawl()
- self.spider = self.crawler.spider
+ self.crawler = get_crawler(Spider)
+ self.spider = self.crawler._create_spider("example.com")
self.tmpdir = tempfile.mkdtemp()
self.request = Request("http://www.example.com", headers={"User-Agent": "test"})
self.response = Response(
@@ -35,6 +31,7 @@ class _BaseTest(unittest.TestCase):
body=b"test body",
status=202,
)
+ self.crawler.stats.open_spider(self.spider)
def tearDown(self):
self.crawler.stats.close_spider(self.spider, "")
@@ -569,3 +566,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
res2 = self._process_requestresponse(mw, req0, None)
self.assertEqualResponse(res1, res2)
assert "cached" in res2.flags
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 4a579c061..9dad056de 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,10 +1,9 @@
from gzip import GzipFile
from io import BytesIO
from pathlib import Path
+from unittest import SkipTest, TestCase
from warnings import catch_warnings
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
from w3lib.encoding import resolve_encoding
from scrapy.downloadermiddlewares.httpcompression import (
@@ -14,10 +13,10 @@ from scrapy.downloadermiddlewares.httpcompression import (
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
+from scrapy.spiders import Spider
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
from tests import tests_datadir
-from tests.spiders import NoRequestsSpider
SAMPLEDIR = Path(tests_datadir, "compressed")
@@ -39,13 +38,12 @@ FORMAT = {
}
-class HttpCompressionTest(unittest.TestCase):
- @inlineCallbacks
+class HttpCompressionTest(TestCase):
def setUp(self):
- self.crawler = get_crawler(NoRequestsSpider)
- yield self.crawler.crawl()
- self.spider = self.crawler.spider
+ self.crawler = get_crawler(Spider)
+ self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
+ self.crawler.stats.open_spider(self.spider)
def _getresponse(self, coding):
if coding not in FORMAT:
@@ -133,7 +131,7 @@ class HttpCompressionTest(unittest.TestCase):
try:
import brotli # noqa: F401
except ImportError:
- raise unittest.SkipTest("no brotli")
+ raise SkipTest("no brotli")
response = self._getresponse("br")
request = response.request
self.assertEqual(response.headers["Content-Encoding"], b"br")
@@ -148,7 +146,7 @@ class HttpCompressionTest(unittest.TestCase):
try:
import zstandard # noqa: F401
except ImportError:
- raise unittest.SkipTest("no zstd support (zstandard)")
+ raise SkipTest("no zstd support (zstandard)")
raw_content = None
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
@@ -376,15 +374,13 @@ class HttpCompressionTest(unittest.TestCase):
self.assertStatsEqual("httpcompression/response_bytes", None)
-class HttpCompressionSubclassTest(unittest.TestCase):
- @inlineCallbacks
+class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):
class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
def __init__(self):
super().__init__()
- crawler = get_crawler(NoRequestsSpider)
- yield crawler.crawl()
+ crawler = get_crawler(Spider)
with catch_warnings(record=True) as caught_warnings:
HttpCompressionMiddlewareSubclass.from_crawler(crawler)
messages = tuple(
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index f94958ff7..97ae1e29a 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,9 +1,9 @@
import logging
+import unittest
import warnings
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from twisted.internet.error import (
ConnectError,
ConnectionDone,
@@ -11,7 +11,6 @@ from twisted.internet.error import (
DNSLookupError,
TCPTimedOutError,
)
-from twisted.trial import unittest
from twisted.web.client import ResponseFailed
from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
@@ -20,15 +19,12 @@ from scrapy.http import Request, Response
from scrapy.settings.default_settings import RETRY_EXCEPTIONS
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
class RetryTest(unittest.TestCase):
- @inlineCallbacks
def setUp(self):
- self.crawler = get_crawler(NoRequestsSpider)
- yield self.crawler.crawl()
- self.spider = self.crawler.spider
+ self.crawler = get_crawler(Spider)
+ self.spider = self.crawler._create_spider("foo")
self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw.max_retry_times = 2
@@ -164,19 +160,16 @@ class RetryTest(unittest.TestCase):
class MaxRetryTimesTest(unittest.TestCase):
invalid_url = "http://www.scrapytest.org/invalid_url"
- @inlineCallbacks
def get_spider_and_middleware(self, settings=None):
- crawler = get_crawler(NoRequestsSpider, settings or {})
- yield crawler.crawl()
- spider = crawler.spider
+ crawler = get_crawler(Spider, settings or {})
+ spider = crawler._create_spider("foo")
middleware = RetryMiddleware.from_crawler(crawler)
return spider, middleware
- @inlineCallbacks
def test_with_settings_zero(self):
max_retry_times = 0
settings = {"RETRY_TIMES": max_retry_times}
- spider, middleware = yield self.get_spider_and_middleware(settings)
+ spider, middleware = self.get_spider_and_middleware(settings)
req = Request(self.invalid_url)
self._test_retry(
req,
@@ -186,10 +179,9 @@ class MaxRetryTimesTest(unittest.TestCase):
middleware=middleware,
)
- @inlineCallbacks
def test_with_metakey_zero(self):
max_retry_times = 0
- spider, middleware = yield self.get_spider_and_middleware()
+ spider, middleware = self.get_spider_and_middleware()
meta = {"max_retry_times": max_retry_times}
req = Request(self.invalid_url, meta=meta)
self._test_retry(
@@ -200,11 +192,10 @@ class MaxRetryTimesTest(unittest.TestCase):
middleware=middleware,
)
- @inlineCallbacks
def test_without_metakey(self):
max_retry_times = 5
settings = {"RETRY_TIMES": max_retry_times}
- spider, middleware = yield self.get_spider_and_middleware(settings)
+ spider, middleware = self.get_spider_and_middleware(settings)
req = Request(self.invalid_url)
self._test_retry(
req,
@@ -214,7 +205,6 @@ class MaxRetryTimesTest(unittest.TestCase):
middleware=middleware,
)
- @inlineCallbacks
def test_with_metakey_greater(self):
meta_max_retry_times = 3
middleware_max_retry_times = 2
@@ -223,7 +213,7 @@ class MaxRetryTimesTest(unittest.TestCase):
req2 = Request(self.invalid_url)
settings = {"RETRY_TIMES": middleware_max_retry_times}
- spider, middleware = yield self.get_spider_and_middleware(settings)
+ spider, middleware = self.get_spider_and_middleware(settings)
self._test_retry(
req1,
@@ -240,7 +230,6 @@ class MaxRetryTimesTest(unittest.TestCase):
middleware=middleware,
)
- @inlineCallbacks
def test_with_metakey_lesser(self):
meta_max_retry_times = 4
middleware_max_retry_times = 5
@@ -249,7 +238,7 @@ class MaxRetryTimesTest(unittest.TestCase):
req2 = Request(self.invalid_url)
settings = {"RETRY_TIMES": middleware_max_retry_times}
- spider, middleware = yield self.get_spider_and_middleware(settings)
+ spider, middleware = self.get_spider_and_middleware(settings)
self._test_retry(
req1,
@@ -266,10 +255,9 @@ class MaxRetryTimesTest(unittest.TestCase):
middleware=middleware,
)
- @inlineCallbacks
def test_with_dont_retry(self):
max_retry_times = 4
- spider, middleware = yield self.get_spider_and_middleware()
+ spider, middleware = self.get_spider_and_middleware()
meta = {
"max_retry_times": max_retry_times,
"dont_retry": True,
@@ -304,16 +292,13 @@ class MaxRetryTimesTest(unittest.TestCase):
class GetRetryRequestTest(unittest.TestCase):
- @inlineCallbacks
def get_spider(self, settings=None):
- crawler = get_crawler(NoRequestsSpider, settings or {})
- yield crawler.crawl()
- return crawler.spider
+ crawler = get_crawler(Spider, settings or {})
+ return crawler._create_spider("foo")
- @inlineCallbacks
def test_basic_usage(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
with LogCapture() as log:
new_request = get_retry_request(
request,
@@ -337,10 +322,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_max_retries_reached(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
max_retry_times = 0
with LogCapture() as log:
new_request = get_retry_request(
@@ -361,10 +345,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_one_retry(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
with LogCapture() as log:
new_request = get_retry_request(
request,
@@ -389,9 +372,8 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_two_retries(self):
- spider = yield self.get_spider()
+ spider = self.get_spider()
request = Request("https://example.com")
new_request = request
max_retry_times = 2
@@ -445,10 +427,9 @@ class GetRetryRequestTest(unittest.TestCase):
with self.assertRaises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
- @inlineCallbacks
def test_max_retry_times_setting(self):
max_retry_times = 0
- spider = yield self.get_spider({"RETRY_TIMES": max_retry_times})
+ spider = self.get_spider({"RETRY_TIMES": max_retry_times})
request = Request("https://example.com")
new_request = get_retry_request(
request,
@@ -456,10 +437,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
self.assertEqual(new_request, None)
- @inlineCallbacks
def test_max_retry_times_meta(self):
max_retry_times = 0
- spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1})
+ spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1})
meta = {"max_retry_times": max_retry_times}
request = Request("https://example.com", meta=meta)
new_request = get_retry_request(
@@ -468,10 +448,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
self.assertEqual(new_request, None)
- @inlineCallbacks
def test_max_retry_times_argument(self):
max_retry_times = 0
- spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1})
+ spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1})
meta = {"max_retry_times": max_retry_times + 1}
request = Request("https://example.com", meta=meta)
new_request = get_retry_request(
@@ -481,10 +460,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
self.assertEqual(new_request, None)
- @inlineCallbacks
def test_priority_adjust_setting(self):
priority_adjust = 1
- spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust})
+ spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust})
request = Request("https://example.com")
new_request = get_retry_request(
request,
@@ -492,10 +470,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
self.assertEqual(new_request.priority, priority_adjust)
- @inlineCallbacks
def test_priority_adjust_argument(self):
priority_adjust = 1
- spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1})
+ spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1})
request = Request("https://example.com")
new_request = get_retry_request(
request,
@@ -504,10 +481,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
self.assertEqual(new_request.priority, priority_adjust)
- @inlineCallbacks
def test_log_extra_retry_success(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
with LogCapture(attributes=("spider",)) as log:
get_retry_request(
request,
@@ -515,10 +491,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
log.check_present(spider)
- @inlineCallbacks
def test_log_extra_retries_exceeded(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
with LogCapture(attributes=("spider",)) as log:
get_retry_request(
request,
@@ -527,10 +502,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
log.check_present(spider)
- @inlineCallbacks
def test_reason_string(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = "because"
with LogCapture() as log:
get_retry_request(
@@ -550,10 +524,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_reason_builtin_exception(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = NotImplementedError()
expected_reason_string = "builtins.NotImplementedError"
with LogCapture() as log:
@@ -576,10 +549,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_reason_builtin_exception_class(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = NotImplementedError
expected_reason_string = "builtins.NotImplementedError"
with LogCapture() as log:
@@ -602,10 +574,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_reason_custom_exception(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = IgnoreRequest()
expected_reason_string = "scrapy.exceptions.IgnoreRequest"
with LogCapture() as log:
@@ -628,10 +599,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_reason_custom_exception_class(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = IgnoreRequest
expected_reason_string = "scrapy.exceptions.IgnoreRequest"
with LogCapture() as log:
@@ -654,11 +624,10 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_custom_logger(self):
logger = logging.getLogger("custom-logger")
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = "because"
with LogCapture() as log:
get_retry_request(
@@ -675,10 +644,9 @@ class GetRetryRequestTest(unittest.TestCase):
)
)
- @inlineCallbacks
def test_custom_stats_key(self):
request = Request("https://example.com")
- spider = yield self.get_spider()
+ spider = self.get_spider()
expected_reason = "because"
stats_key = "custom_retry"
get_retry_request(
@@ -692,3 +660,7 @@ class GetRetryRequestTest(unittest.TestCase):
f"{stats_key}/reason_count/{expected_reason}",
):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 55ae0c2b7..39dfe9ab5 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,29 +1,27 @@
import warnings
from itertools import product
-
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
+from unittest import TestCase
from scrapy.downloadermiddlewares.stats import DownloaderStats
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
+from scrapy.spiders import Spider
from scrapy.utils.response import response_httprepr
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
class MyException(Exception):
pass
-class TestDownloaderStats(unittest.TestCase):
- @inlineCallbacks
+class TestDownloaderStats(TestCase):
def setUp(self):
- self.crawler = get_crawler(NoRequestsSpider)
- yield self.crawler.crawl()
- self.spider = self.crawler.spider
+ self.crawler = get_crawler(Spider)
+ self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = DownloaderStats(self.crawler.stats)
+ self.crawler.stats.open_spider(self.spider)
+
self.req = Request("http://scrapytest.org")
self.res = Response("scrapytest.org", status=400)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 1ba6125b2..aa0975555 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -2,25 +2,22 @@ import hashlib
import shutil
import sys
import tempfile
+import unittest
from pathlib import Path
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
from scrapy.core.scheduler import Scheduler
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
+from tests.spiders import SimpleSpider
-@inlineCallbacks
def _get_dupefilter(*, crawler=None, settings=None, open=True):
if crawler is None:
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
dupefilter = scheduler.df
if open:
@@ -47,51 +44,41 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter):
class DirectDupeFilter:
method = "n/a"
- def open(self):
- pass
-
class RFPDupeFilterTest(unittest.TestCase):
- @inlineCallbacks
def test_df_from_crawler_scheduler(self):
settings = {
"DUPEFILTER_DEBUG": True,
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
self.assertEqual(scheduler.df.method, "from_crawler")
- @inlineCallbacks
def test_df_from_settings_scheduler(self):
settings = {
"DUPEFILTER_DEBUG": True,
"DUPEFILTER_CLASS": FromSettingsRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
self.assertEqual(scheduler.df.method, "from_settings")
- @inlineCallbacks
def test_df_direct_scheduler(self):
settings = {
"DUPEFILTER_CLASS": DirectDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertEqual(scheduler.df.method, "n/a")
- @inlineCallbacks
def test_filter(self):
- dupefilter = yield _get_dupefilter()
+ dupefilter = _get_dupefilter()
r1 = Request("http://scrapytest.org/1")
r2 = Request("http://scrapytest.org/2")
r3 = Request("http://scrapytest.org/2")
@@ -104,14 +91,13 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.close("finished")
- @inlineCallbacks
def test_dupefilter_path(self):
r1 = Request("http://scrapytest.org/1")
r2 = Request("http://scrapytest.org/2")
path = tempfile.mkdtemp()
try:
- df = yield _get_dupefilter(settings={"JOBDIR": path}, open=False)
+ df = _get_dupefilter(settings={"JOBDIR": path}, open=False)
try:
df.open()
assert not df.request_seen(r1)
@@ -119,7 +105,7 @@ class RFPDupeFilterTest(unittest.TestCase):
finally:
df.close("finished")
- df2 = yield _get_dupefilter(settings={"JOBDIR": path}, open=False)
+ df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False)
assert df != df2
try:
df2.open()
@@ -131,13 +117,12 @@ class RFPDupeFilterTest(unittest.TestCase):
finally:
shutil.rmtree(path)
- @inlineCallbacks
def test_request_fingerprint(self):
"""Test if customization of request_fingerprint method will change
output of request_seen.
"""
- dupefilter = yield _get_dupefilter()
+ dupefilter = _get_dupefilter()
r1 = Request("http://scrapytest.org/index.html")
r2 = Request("http://scrapytest.org/INDEX.html")
@@ -153,14 +138,13 @@ class RFPDupeFilterTest(unittest.TestCase):
return fp.digest()
settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter}
- case_insensitive_dupefilter = yield _get_dupefilter(settings=settings)
+ case_insensitive_dupefilter = _get_dupefilter(settings=settings)
assert not case_insensitive_dupefilter.request_seen(r1)
assert case_insensitive_dupefilter.request_seen(r2)
case_insensitive_dupefilter.close("finished")
- @inlineCallbacks
def test_seenreq_newlines(self):
"""Checks against adding duplicate \r to
line endings on Windows platforms."""
@@ -168,8 +152,7 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request("http://scrapytest.org/1")
path = tempfile.mkdtemp()
- crawler = get_crawler(NoRequestsSpider, settings_dict={"JOBDIR": path})
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict={"JOBDIR": path})
try:
scheduler = Scheduler.from_crawler(crawler)
df = scheduler.df
@@ -178,10 +161,7 @@ class RFPDupeFilterTest(unittest.TestCase):
df.close("finished")
with Path(path, "requests.seen").open("rb") as seen_file:
- try:
- line = next(seen_file).decode()
- except StopIteration:
- return
+ line = next(seen_file).decode()
assert not line.endswith("\r\r\n")
if sys.platform == "win32":
assert line.endswith("\r\n")
@@ -191,7 +171,6 @@ class RFPDupeFilterTest(unittest.TestCase):
finally:
shutil.rmtree(path)
- @inlineCallbacks
def test_log(self):
with LogCapture() as log:
settings = {
@@ -199,10 +178,9 @@ class RFPDupeFilterTest(unittest.TestCase):
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
- spider = crawler.spider
- dupefilter = yield _get_dupefilter(crawler=crawler)
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request("http://scrapytest.org/index.html")
r2 = Request("http://scrapytest.org/index.html")
@@ -222,7 +200,6 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.close("finished")
- @inlineCallbacks
def test_log_debug(self):
with LogCapture() as log:
settings = {
@@ -230,10 +207,9 @@ class RFPDupeFilterTest(unittest.TestCase):
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
- spider = crawler.spider
- dupefilter = yield _get_dupefilter(crawler=crawler)
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request("http://scrapytest.org/index.html")
r2 = Request(
@@ -263,17 +239,15 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.close("finished")
- @inlineCallbacks
def test_log_debug_default_dupefilter(self):
with LogCapture() as log:
settings = {
"DUPEFILTER_DEBUG": True,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
- spider = crawler.spider
- dupefilter = yield _get_dupefilter(crawler=crawler)
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request("http://scrapytest.org/index.html")
r2 = Request(
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 5deae5146..8d7afb6a1 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -420,16 +420,12 @@ class EngineTest(unittest.TestCase):
@defer.inlineCallbacks
def test_close_downloader(self):
- crawler = get_crawler(TestSpider)
- yield crawler.crawl()
- e = ExecutionEngine(crawler, lambda _: None)
+ e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.close()
@defer.inlineCallbacks
def test_start_already_running_exception(self):
- crawler = get_crawler(TestSpider)
- yield crawler.crawl()
- e = ExecutionEngine(crawler, lambda _: None)
+ e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
try:
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index ce8452b5f..6e5fb0325 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -1,8 +1,6 @@
import datetime
import typing
-
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
+import unittest
from scrapy.crawler import Crawler
from scrapy.extensions.periodic_log import PeriodicLog
@@ -61,44 +59,33 @@ class TestExtPeriodicLog(PeriodicLog):
self.stats._stats = stats_dump_2
-@inlineCallbacks
def extension(settings=None):
- crawler = Crawler(
- MetaSpider,
- settings=settings,
- )
- yield crawler.crawl()
+ crawler = Crawler(MetaSpider, settings=settings)
+ crawler._load_settings()
return TestExtPeriodicLog.from_crawler(crawler)
class TestPeriodicLog(unittest.TestCase):
- @inlineCallbacks
def test_extension_enabled(self):
# Expected that settings for this extension loaded succesfully
# And on certain conditions - extension raising NotConfigured
# "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
# due to TypeError exception from settings.getdict
- assert (yield extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}))
+ assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})
# "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True}
# due to JSONDecodeError(ValueError) exception from settings.getdict
- assert (
- yield extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})
- )
+ assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60})
# The ame for PERIODIC_LOG_DELTA:
- assert (yield extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}))
- assert (
- yield extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
- )
+ assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
+ assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
- @inlineCallbacks
def test_log_delta(self):
- @inlineCallbacks
def emulate(settings=None):
spider = MetaSpider()
- ext = yield extension(settings)
+ ext = extension(settings)
ext.spider_opened(spider)
ext.set_a()
a = ext.log_delta()
@@ -107,9 +94,8 @@ class TestPeriodicLog(unittest.TestCase):
ext.spider_closed(spider, reason="finished")
return ext, a, b
- @inlineCallbacks
def check(settings: dict, condition: typing.Callable):
- ext, a, b = yield emulate(settings)
+ ext, a, b = emulate(settings)
assert list(a["delta"].keys()) == [
k for k, v in ext.stats._stats.items() if condition(k, v)
]
@@ -118,49 +104,45 @@ class TestPeriodicLog(unittest.TestCase):
]
# Including all
- yield check(
- {"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))
- )
+ check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)))
# include:
- yield check(
+ check(
{"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}},
lambda k, v: isinstance(v, (int, float)) and "downloader/" in k,
)
# include multiple
- yield check(
+ check(
{"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},
lambda k, v: isinstance(v, (int, float))
and ("downloader/" in k or "scheduler/" in k),
)
# exclude
- yield check(
+ check(
{"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}},
lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k,
)
# exclude multiple
- yield check(
+ check(
{"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},
lambda k, v: isinstance(v, (int, float))
and ("downloader/" not in k and "scheduler/" not in k),
)
# include exclude combined
- yield check(
+ check(
{"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},
lambda k, v: isinstance(v, (int, float))
and ("downloader/" in k and "bytes" not in k),
)
- @inlineCallbacks
def test_log_stats(self):
- @inlineCallbacks
def emulate(settings=None):
spider = MetaSpider()
- ext = yield extension(settings)
+ ext = extension(settings)
ext.spider_opened(spider)
ext.set_a()
a = ext.log_crawler_stats()
@@ -169,9 +151,8 @@ class TestPeriodicLog(unittest.TestCase):
ext.spider_closed(spider, reason="finished")
return ext, a, b
- @inlineCallbacks
def check(settings: dict, condition: typing.Callable):
- ext, a, b = yield emulate(settings)
+ ext, a, b = emulate(settings)
assert list(a["stats"].keys()) == [
k for k, v in ext.stats._stats.items() if condition(k, v)
]
@@ -180,34 +161,35 @@ class TestPeriodicLog(unittest.TestCase):
]
# Including all
- yield check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)
+ check({"PERIODIC_LOG_STATS": True}, lambda k, v: True)
# include:
- yield check(
+ check(
{"PERIODIC_LOG_STATS": {"include": ["downloader/"]}},
lambda k, v: "downloader/" in k,
)
# include multiple
- yield check(
+ check(
{"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}},
lambda k, v: "downloader/" in k or "scheduler/" in k,
)
# exclude
- yield check(
+ check(
{"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}},
lambda k, v: "downloader/" not in k,
)
# exclude multiple
- yield check(
+ check(
{"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}},
lambda k, v: "downloader/" not in k and "scheduler/" not in k,
)
# include exclude combined
- yield check(
+ check(
{"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},
lambda k, v: "downloader/" in k and "bytes" not in k,
)
+ #
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 6f35510b7..9fd680e9f 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -9,7 +9,7 @@ from scrapy.utils.test import get_crawler
class TelnetExtensionTest(unittest.TestCase):
def _get_console_and_portal(self, settings=None):
- crawler = get_crawler(settings_dict=settings, disable_telnet=False)
+ crawler = get_crawler(settings_dict=settings)
console = TelnetConsole(crawler)
# This function has some side effects we don't need for this test
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 875abdb1f..6b82974fa 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -26,7 +26,6 @@ import lxml.etree
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from w3lib.url import file_uri_to_path, path_to_file_uri
from zope.interface import implementer
@@ -52,7 +51,7 @@ from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto
from tests.mockserver import MockFTPServer, MockServer
-from tests.spiders import ItemSpider, NoRequestsSpider
+from tests.spiders import ItemSpider
def path_to_url(path):
@@ -2748,14 +2747,13 @@ class FeedExporterSignalsTest(unittest.TestCase):
d.callback(None)
return d
- @inlineCallbacks
def run_signaled_feed_exporter(
self, feed_exporter_signal_handler, feed_slot_signal_handler
):
- crawler = get_crawler(NoRequestsSpider, settings_dict=self.settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=self.settings)
feed_exporter = FeedExporter.from_crawler(crawler)
- spider = crawler.spider
+ spider = scrapy.Spider("default")
+ spider.crawler = crawler
crawler.signals.connect(
feed_exporter_signal_handler,
signal=signals.feed_exporter_closed,
@@ -2766,26 +2764,24 @@ class FeedExporterSignalsTest(unittest.TestCase):
feed_exporter.open_spider(spider)
for item in self.items:
feed_exporter.item_scraped(item, spider)
- yield defer.ensureDeferred(feed_exporter.close_spider(spider))
+ defer.ensureDeferred(feed_exporter.close_spider(spider))
- @inlineCallbacks
def test_feed_exporter_signals_sent(self):
self.feed_exporter_closed_received = False
self.feed_slot_closed_received = False
- yield self.run_signaled_feed_exporter(
+ self.run_signaled_feed_exporter(
self.feed_exporter_closed_signal_handler,
self.feed_slot_closed_signal_handler,
)
self.assertTrue(self.feed_slot_closed_received)
self.assertTrue(self.feed_exporter_closed_received)
- @inlineCallbacks
def test_feed_exporter_signals_sent_deferred(self):
self.feed_exporter_closed_received = False
self.feed_slot_closed_received = False
- yield self.run_signaled_feed_exporter(
+ self.run_signaled_feed_exporter(
self.feed_exporter_closed_signal_handler_deferred,
self.feed_slot_closed_signal_handler_deferred,
)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0d769f29b..bf96f17b6 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -15,7 +15,6 @@ import attr
import pytest
from itemadapter import ItemAdapter
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from scrapy.http import Request, Response
@@ -36,18 +35,15 @@ from scrapy.utils.test import (
skip_if_no_boto,
)
from tests.mockserver import MockFTPServer
-from tests.spiders import NoRequestsSpider
from .test_pipeline_media import _mocked_download_func
class FilesPipelineTestCase(unittest.TestCase):
- @inlineCallbacks
def setUp(self):
self.tempdir = mkdtemp()
settings_dict = {"FILES_STORE": self.tempdir}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func
self.pipeline.open_spider(None)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 8a13ea552..d655eb128 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -14,11 +14,11 @@ from scrapy.pipelines.files import FileException
from scrapy.pipelines.images import ImagesPipeline
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
+from scrapy.spiders import Spider
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
try:
from PIL import Image # noqa: imported just to check for the import error
@@ -40,11 +40,10 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
pipeline_class = MediaPipeline
settings = None
- @inlineCallbacks
def setUp(self):
- crawler = get_crawler(NoRequestsSpider, self.settings)
- yield crawler.crawl()
- self.spider = crawler.spider
+ spider_cls = Spider
+ self.spider = spider_cls("media.com")
+ crawler = get_crawler(spider_cls, self.settings)
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider)
@@ -431,14 +430,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
skip = skip_pillow
- @inlineCallbacks
def setUp(self):
settings_dict = {
"IMAGES_STORE": "store-uri",
"IMAGES_THUMBS": {"small": (50, 50)},
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict)
- yield crawler.crawl()
+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(None)
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 54c77eb68..ef9b360c4 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1,10 +1,10 @@
import collections
import shutil
import tempfile
+import unittest
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
+from twisted.trial.unittest import TestCase
from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import Scheduler
@@ -15,7 +15,6 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-from tests.spiders import NoRequestsSpider
MockEngine = collections.namedtuple("MockEngine", ["downloader"])
MockSlot = collections.namedtuple("MockSlot", ["active"])
@@ -310,7 +309,7 @@ class StartUrlsSpider(Spider):
pass
-class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase):
+class TestIntegrationWithDownloaderAwareInMemory(TestCase):
def setUp(self):
self.crawler = get_crawler(
spidercls=StartUrlsSpider,
@@ -337,19 +336,16 @@ class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase):
class TestIncompatibility(unittest.TestCase):
- @inlineCallbacks
def _incompatible(self):
settings = dict(
SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue",
CONCURRENT_REQUESTS_PER_IP=1,
)
- crawler = get_crawler(NoRequestsSpider, settings)
- yield crawler.crawl()
- spider = crawler.spider
+ crawler = get_crawler(Spider, settings)
scheduler = Scheduler.from_crawler(crawler)
+ spider = Spider(name="spider")
scheduler.open(spider)
- @inlineCallbacks
def test_incompatibility(self):
with self.assertRaises(ValueError):
- yield self._incompatible()
+ self._incompatible()
diff --git a/tests/test_spider.py b/tests/test_spider.py
index a88d9b505..00da3d485 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -114,7 +114,7 @@ class SpiderTest(unittest.TestCase):
spider.settings.set("TEST1", "spider_instance", priority="spider")
return spider
- crawler = get_crawler(TestSpider, settings_dict=project_settings)
+ crawler = Crawler(TestSpider, project_settings)
self.assertEqual(crawler.settings.get("TEST1"), "spider")
self.assertEqual(crawler.settings.get("TEST2"), "spider")
self.assertEqual(crawler.settings.get("TEST3"), "project")
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 2fcb5b364..1d5a887cc 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,9 +1,9 @@
import logging
+from unittest import TestCase
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial.unittest import TestCase
+from twisted.trial.unittest import TestCase as TrialTestCase
from scrapy.http import Request, Response
from scrapy.settings import Settings
@@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-from tests.spiders import MockServerSpider, NoRequestsSpider
+from tests.spiders import MockServerSpider
class _HttpErrorSpider(MockServerSpider):
@@ -59,11 +59,9 @@ def _responses(request, status_codes):
class TestHttpErrorMiddleware(TestCase):
- @inlineCallbacks
def setUp(self):
- crawler = get_crawler(NoRequestsSpider)
- yield crawler.crawl()
- self.spider = crawler.spider
+ crawler = get_crawler(Spider)
+ self.spider = Spider.from_crawler(crawler, name="foo")
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request("http://scrapytest.org")
self.res200, self.res404 = _responses(self.req, [200, 404])
@@ -173,7 +171,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.assertIsNone(mw.process_spider_input(res402, self.spider))
-class TestHttpErrorMiddlewareIntegrational(TestCase):
+class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index ed00c0a6b..ea45b7698 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,21 +1,17 @@
import warnings
+from unittest import TestCase
from urllib.parse import urlparse
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
-
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class TestOffsiteMiddleware(unittest.TestCase):
- @inlineCallbacks
+class TestOffsiteMiddleware(TestCase):
def setUp(self):
crawler = get_crawler(Spider)
- yield crawler.crawl(**self._get_spiderargs())
- self.spider = crawler.spider
+ self.spider = crawler._create_spider(**self._get_spiderargs())
self.mw = OffsiteMiddleware.from_crawler(crawler)
self.mw.spider_opened(self.spider)
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index a3c284484..9111e4c82 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -1,23 +1,21 @@
+from unittest import TestCase
+
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
+from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
-class TestUrlLengthMiddleware(unittest.TestCase):
- @inlineCallbacks
+class TestUrlLengthMiddleware(TestCase):
def setUp(self):
self.maxlength = 25
settings = Settings({"URLLENGTH_LIMIT": self.maxlength})
- crawler = get_crawler(NoRequestsSpider)
- yield crawler.crawl()
- self.spider = crawler.spider
+ crawler = get_crawler(Spider)
+ self.spider = crawler._create_spider("foo")
self.stats = crawler.stats
self.mw = UrlLengthMiddleware.from_settings(settings)
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 2117e65b1..eae744df5 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,10 +1,9 @@
import logging
import sys
+import unittest
from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
-from twisted.trial import unittest
from scrapy.extensions import telnet
from scrapy.utils.log import (
@@ -14,7 +13,6 @@ from scrapy.utils.log import (
failure_to_exc_info,
)
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
class FailureToExcInfoTest(unittest.TestCase):
@@ -62,7 +60,6 @@ class TopLevelFormatterTest(unittest.TestCase):
class LogCounterHandlerTest(unittest.TestCase):
- @inlineCallbacks
def setUp(self):
settings = {"LOG_LEVEL": "WARNING"}
if not telnet.TWISTED_CONCH_AVAILABLE:
@@ -71,8 +68,7 @@ class LogCounterHandlerTest(unittest.TestCase):
self.logger = logging.getLogger("test")
self.logger.setLevel(logging.NOTSET)
self.logger.propagate = False
- self.crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield self.crawler.crawl()
+ self.crawler = get_crawler(settings_dict=settings)
self.handler = LogCounterHandler(self.crawler)
self.logger.addHandler(self.handler)
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 9ca9faa0c..e6d1abe3f 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,12 +1,11 @@
import json
+import unittest
import warnings
from hashlib import sha1
from typing import Dict, Mapping, Optional, Tuple, Union
from weakref import WeakKeyDictionary
import pytest
-from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
from w3lib.url import canonicalize_url
from scrapy.http import Request
@@ -23,7 +22,6 @@ from scrapy.utils.request import (
request_to_curl,
)
from scrapy.utils.test import get_crawler
-from tests.spiders import NoRequestsSpider
class UtilsRequestTest(unittest.TestCase):
@@ -451,18 +449,15 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
)
self.assertEqual(fp, old_fp)
- @inlineCallbacks
def test_component_backward_compatibility(self):
for request_object in REQUEST_OBJECTS_TO_TEST:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
- crawler = get_crawler(NoRequestsSpider, prevent_warnings=False)
- yield crawler.crawl()
+ crawler = get_crawler(prevent_warnings=False)
fp = crawler.request_fingerprinter.fingerprint(request_object)
old_fp = request_fingerprint_2_6(request_object)
self.assertEqual(fp.hex(), old_fp)
- @inlineCallbacks
def test_custom_component_backward_compatibility(self):
"""Tests that the backward-compatible request fingerprinting class featured
in the documentation is indeed backward compatible and does not cause a
@@ -485,8 +480,7 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
fp = crawler.request_fingerprinter.fingerprint(request_object)
old_fp = request_fingerprint_2_6(request_object)
self.assertEqual(fp.hex(), old_fp)
@@ -494,11 +488,9 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
class RequestFingerprinterTestCase(unittest.TestCase):
- @inlineCallbacks
def test_default_implementation(self):
with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(NoRequestsSpider, prevent_warnings=False)
- yield crawler.crawl()
+ crawler = get_crawler(prevent_warnings=False)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
@@ -506,14 +498,12 @@ class RequestFingerprinterTestCase(unittest.TestCase):
)
self.assertTrue(logged_warnings)
- @inlineCallbacks
def test_deprecated_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
}
with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
@@ -521,14 +511,12 @@ class RequestFingerprinterTestCase(unittest.TestCase):
)
self.assertTrue(logged_warnings)
- @inlineCallbacks
def test_recommended_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
@@ -536,18 +524,15 @@ class RequestFingerprinterTestCase(unittest.TestCase):
)
self.assertFalse(logged_warnings)
- @inlineCallbacks
def test_unknown_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
}
with self.assertRaises(ValueError):
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ get_crawler(settings_dict=settings)
class CustomRequestFingerprinterTestCase(unittest.TestCase):
- @inlineCallbacks
def test_include_headers(self):
class RequestFingerprinter:
def fingerprint(self, request):
@@ -556,8 +541,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
@@ -565,7 +549,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
fp2 = crawler.request_fingerprinter.fingerprint(r2)
self.assertNotEqual(fp1, fp2)
- @inlineCallbacks
def test_dont_canonicalize(self):
class RequestFingerprinter:
cache = WeakKeyDictionary()
@@ -580,8 +563,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
r1 = Request("http://www.example.com?a=1&a=2")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
@@ -589,7 +571,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
fp2 = crawler.request_fingerprinter.fingerprint(r2)
self.assertNotEqual(fp1, fp2)
- @inlineCallbacks
def test_meta(self):
class RequestFingerprinter:
def fingerprint(self, request):
@@ -600,8 +581,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
@@ -616,7 +596,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
self.assertNotEqual(fp2, fp4)
self.assertEqual(fp2, fp3)
- @inlineCallbacks
def test_from_crawler(self):
class RequestFingerprinter:
@classmethod
@@ -633,14 +612,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
"FINGERPRINT": b"fingerprint",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
self.assertEqual(fingerprint, settings["FINGERPRINT"])
- @inlineCallbacks
def test_from_settings(self):
class RequestFingerprinter:
@classmethod
@@ -657,14 +634,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
"FINGERPRINT": b"fingerprint",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
self.assertEqual(fingerprint, settings["FINGERPRINT"])
- @inlineCallbacks
def test_from_crawler_and_settings(self):
class RequestFingerprinter:
# This method is ignored due to the presence of from_crawler
@@ -686,8 +661,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
"FINGERPRINT": b"fingerprint",
}
- crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
- yield crawler.crawl()
+ crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
@@ -754,3 +728,7 @@ class RequestToCurlTest(unittest.TestCase):
" --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
)
self._test_request(request_object, expected_curl_command)
+
+
+if __name__ == "__main__":
+ unittest.main()