Scrapy, a fast high-level web crawling & scraping framework for Python.
Convert tests that fail with testfixtures 12.0.0 (#7545)
Andrey Rakhmatullin90deebe75e49
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex d5ad59346..6a054ded5 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -1,17 +1,19 @@from __future__ import annotationsimport logging+from typing import TYPE_CHECKINGimport pytest-from testfixtures import LogCapturefrom scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddlewarefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler-from tests.mockserver.http import MockServerfrom tests.spiders import MockServerSpider-from tests.utils.decorators import inline_callbacks_test+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from tests.mockserver.http import MockServerclass _HttpErrorSpider(MockServerSpider):@@ -192,65 +194,66 @@ class TestHttpErrorMiddlewareHandleAll:class TestHttpErrorMiddlewareIntegrational:- @classmethod- def setup_class(cls):- cls.mockserver = MockServer()- cls.mockserver.__enter__()-- @classmethod- def teardown_class(cls):- cls.mockserver.__exit__(None, None, None)-- @inline_callbacks_test- def test_middleware_works(self):+ @coroutine_test+ async def test_middleware_works(self, mockserver: MockServer) -> None:crawler = get_crawler(_HttpErrorSpider)- yield crawler.crawl(mockserver=self.mockserver)+ await crawler.crawl_async(mockserver=mockserver)+ assert isinstance(crawler.spider, _HttpErrorSpider)assert not crawler.spider.skippedassert crawler.spider.parsed == {"200"}assert crawler.spider.failed == {"404", "402", "500"}+ assert crawler.statsget_value = crawler.stats.get_valueassert get_value("httperror/response_ignored_count") == 3assert get_value("httperror/response_ignored_status_count/404") == 1assert get_value("httperror/response_ignored_status_count/402") == 1assert get_value("httperror/response_ignored_status_count/500") == 1- @inline_callbacks_test- def test_logging(self):+ @coroutine_test+ async def test_logging(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:crawler = get_crawler(_HttpErrorSpider)- with LogCapture() as log:- yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})+ with caplog.at_level(logging.INFO):+ await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402})+ assert isinstance(crawler.spider, _HttpErrorSpider)assert crawler.spider.parsed == {"200", "402"}assert crawler.spider.skipped == {"402"}assert crawler.spider.failed == {"404", "500"}- assert "Ignoring response <404" in str(log)- assert "Ignoring response <500" in str(log)- assert "Ignoring response <200" not in str(log)- assert "Ignoring response <402" not in str(log)+ assert "Ignoring response <404" in caplog.text+ assert "Ignoring response <500" in caplog.text+ assert "Ignoring response <200" not in caplog.text+ assert "Ignoring response <402" not in caplog.text- @inline_callbacks_test- def test_logging_level(self):+ @coroutine_test+ async def test_logging_level(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:# HttpError logs ignored responses with level INFOcrawler = get_crawler(_HttpErrorSpider)- with LogCapture(level=logging.INFO) as log:- yield crawler.crawl(mockserver=self.mockserver)+ with caplog.at_level(logging.INFO):+ await crawler.crawl_async(mockserver=mockserver)+ assert isinstance(crawler.spider, _HttpErrorSpider)assert crawler.spider.parsed == {"200"}assert crawler.spider.failed == {"404", "402", "500"}- assert "Ignoring response <402" in str(log)- assert "Ignoring response <404" in str(log)- assert "Ignoring response <500" in str(log)- assert "Ignoring response <200" not in str(log)+ assert "Ignoring response <402" in caplog.text+ assert "Ignoring response <404" in caplog.text+ assert "Ignoring response <500" in caplog.text+ assert "Ignoring response <200" not in caplog.text# with level WARNING, we shouldn't capture anything from HttpError+ caplog.clear()crawler = get_crawler(_HttpErrorSpider)- with LogCapture(level=logging.WARNING) as log:- yield crawler.crawl(mockserver=self.mockserver)+ with caplog.at_level(logging.WARNING):+ await crawler.crawl_async(mockserver=mockserver)+ assert isinstance(crawler.spider, _HttpErrorSpider)assert crawler.spider.parsed == {"200"}assert crawler.spider.failed == {"404", "402", "500"}- assert "Ignoring response <402" not in str(log)- assert "Ignoring response <404" not in str(log)- assert "Ignoring response <500" not in str(log)- assert "Ignoring response <200" not in str(log)+ assert "Ignoring response <402" not in caplog.text+ assert "Ignoring response <404" not in caplog.text+ assert "Ignoring response <500" not in caplog.text+ assert "Ignoring response <200" not in caplog.textdiff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 318370d40..8e5020022 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -42,33 +42,29 @@ class TestFailureToExcInfo:class TestTopLevelFormatter:- def setup_method(self):- self.handler = LogCapture()- self.handler.addFilter(TopLevelFormatter(["test"]))-- def test_top_level_logger(self):+ def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None:+ caplog.handler.addFilter(TopLevelFormatter(["test"]))logger = logging.getLogger("test")- with self.handler as log:- logger.warning("test log msg")- log.check(("test", "WARNING", "test log msg"))+ logger.warning("test log msg")+ assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples- def test_children_logger(self):+ def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None:+ caplog.handler.addFilter(TopLevelFormatter(["test"]))logger = logging.getLogger("test.test1")- with self.handler as log:- logger.warning("test log msg")- log.check(("test", "WARNING", "test log msg"))+ logger.warning("test log msg")+ assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples- def test_overlapping_name_logger(self):+ def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None:+ caplog.handler.addFilter(TopLevelFormatter(["test"]))logger = logging.getLogger("test2")- with self.handler as log:- logger.warning("test log msg")- log.check(("test2", "WARNING", "test log msg"))+ logger.warning("test log msg")+ assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples- def test_different_name_logger(self):+ def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None:+ caplog.handler.addFilter(TopLevelFormatter(["test"]))logger = logging.getLogger("different")- with self.handler as log:- logger.warning("test log msg")- log.check(("different", "WARNING", "test log msg"))+ logger.warning("test log msg")+ assert ("different", logging.WARNING, "test log msg") in caplog.record_tuplesclass TestLogCounterHandler: