Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Convert tests that fail with testfixtures 12.0.0 (#7545)

Andrey Rakhmatullin90deebe75e49
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index d5ad59346..6a054ded5 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,17 +1,19 @@
from __future__ import annotations
import logging
+from typing import TYPE_CHECKING
import pytest
-from testfixtures import LogCapture
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from tests.mockserver.http import MockServer
class _HttpErrorSpider(MockServerSpider):
@@ -192,65 +194,66 @@ class TestHttpErrorMiddlewareHandleAll:
class TestHttpErrorMiddlewareIntegrational:
- @classmethod
- def setup_class(cls):
- cls.mockserver = MockServer()
- cls.mockserver.__enter__()
-
- @classmethod
- def teardown_class(cls):
- cls.mockserver.__exit__(None, None, None)
-
- @inline_callbacks_test
- def test_middleware_works(self):
+ @coroutine_test
+ async def test_middleware_works(self, mockserver: MockServer) -> None:
crawler = get_crawler(_HttpErrorSpider)
- yield crawler.crawl(mockserver=self.mockserver)
+ await crawler.crawl_async(mockserver=mockserver)
+ assert isinstance(crawler.spider, _HttpErrorSpider)
assert not crawler.spider.skipped
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
+ assert crawler.stats
get_value = crawler.stats.get_value
assert get_value("httperror/response_ignored_count") == 3
assert get_value("httperror/response_ignored_status_count/404") == 1
assert get_value("httperror/response_ignored_status_count/402") == 1
assert get_value("httperror/response_ignored_status_count/500") == 1
- @inline_callbacks_test
- def test_logging(self):
+ @coroutine_test
+ async def test_logging(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
crawler = get_crawler(_HttpErrorSpider)
- with LogCapture() as log:
- yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
+ with caplog.at_level(logging.INFO):
+ await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402})
+ assert isinstance(crawler.spider, _HttpErrorSpider)
assert crawler.spider.parsed == {"200", "402"}
assert crawler.spider.skipped == {"402"}
assert crawler.spider.failed == {"404", "500"}
- assert "Ignoring response <404" in str(log)
- assert "Ignoring response <500" in str(log)
- assert "Ignoring response <200" not in str(log)
- assert "Ignoring response <402" not in str(log)
+ assert "Ignoring response <404" in caplog.text
+ assert "Ignoring response <500" in caplog.text
+ assert "Ignoring response <200" not in caplog.text
+ assert "Ignoring response <402" not in caplog.text
- @inline_callbacks_test
- def test_logging_level(self):
+ @coroutine_test
+ async def test_logging_level(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
# HttpError logs ignored responses with level INFO
crawler = get_crawler(_HttpErrorSpider)
- with LogCapture(level=logging.INFO) as log:
- yield crawler.crawl(mockserver=self.mockserver)
+ with caplog.at_level(logging.INFO):
+ await crawler.crawl_async(mockserver=mockserver)
+ assert isinstance(crawler.spider, _HttpErrorSpider)
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
- assert "Ignoring response <402" in str(log)
- assert "Ignoring response <404" in str(log)
- assert "Ignoring response <500" in str(log)
- assert "Ignoring response <200" not in str(log)
+ assert "Ignoring response <402" in caplog.text
+ assert "Ignoring response <404" in caplog.text
+ assert "Ignoring response <500" in caplog.text
+ assert "Ignoring response <200" not in caplog.text
# with level WARNING, we shouldn't capture anything from HttpError
+ caplog.clear()
crawler = get_crawler(_HttpErrorSpider)
- with LogCapture(level=logging.WARNING) as log:
- yield crawler.crawl(mockserver=self.mockserver)
+ with caplog.at_level(logging.WARNING):
+ await crawler.crawl_async(mockserver=mockserver)
+ assert isinstance(crawler.spider, _HttpErrorSpider)
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
- assert "Ignoring response <402" not in str(log)
- assert "Ignoring response <404" not in str(log)
- assert "Ignoring response <500" not in str(log)
- assert "Ignoring response <200" not in str(log)
+ assert "Ignoring response <402" not in caplog.text
+ assert "Ignoring response <404" not in caplog.text
+ assert "Ignoring response <500" not in caplog.text
+ assert "Ignoring response <200" not in caplog.text
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 318370d40..8e5020022 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -42,33 +42,29 @@ class TestFailureToExcInfo:
class TestTopLevelFormatter:
- def setup_method(self):
- self.handler = LogCapture()
- self.handler.addFilter(TopLevelFormatter(["test"]))
-
- def test_top_level_logger(self):
+ def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+ caplog.handler.addFilter(TopLevelFormatter(["test"]))
logger = logging.getLogger("test")
- with self.handler as log:
- logger.warning("test log msg")
- log.check(("test", "WARNING", "test log msg"))
+ logger.warning("test log msg")
+ assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples
- def test_children_logger(self):
+ def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+ caplog.handler.addFilter(TopLevelFormatter(["test"]))
logger = logging.getLogger("test.test1")
- with self.handler as log:
- logger.warning("test log msg")
- log.check(("test", "WARNING", "test log msg"))
+ logger.warning("test log msg")
+ assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples
- def test_overlapping_name_logger(self):
+ def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+ caplog.handler.addFilter(TopLevelFormatter(["test"]))
logger = logging.getLogger("test2")
- with self.handler as log:
- logger.warning("test log msg")
- log.check(("test2", "WARNING", "test log msg"))
+ logger.warning("test log msg")
+ assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples
- def test_different_name_logger(self):
+ def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+ caplog.handler.addFilter(TopLevelFormatter(["test"]))
logger = logging.getLogger("different")
- with self.handler as log:
- logger.warning("test log msg")
- log.check(("different", "WARNING", "test log msg"))
+ logger.warning("test log msg")
+ assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples
class TestLogCounterHandler: