Scrapy, a fast high-level web crawling & scraping framework for Python.
Address warnings in tests. (#7637)
Andrey Rakhmatullin3f3cb885eddf
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex c0da7dfa4..ebbdf1b98 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -51,6 +51,8 @@ class Contract:results.addSuccess(self.testcase_pre)cb_result = cb(response, **cb_kwargs)if isinstance(cb_result, (AsyncGenerator, CoroutineType)):+ if isinstance(cb_result, CoroutineType):+ cb_result.close()raise TypeError("Contracts don't support async callbacks")return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))@@ -67,6 +69,8 @@ class Contract:def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:cb_result = cb(response, **cb_kwargs)if isinstance(cb_result, (AsyncGenerator, CoroutineType)):+ if isinstance(cb_result, CoroutineType):+ cb_result.close()raise TypeError("Contracts don't support async callbacks")output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))try:diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 47568e656..20e7cb381 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no coverreturn list(walk_modules_iter(path))-def md5sum(file: IO[bytes]) -> str:+def md5sum(file: IO[bytes]) -> str: # pragma: no cover"""Calculate the md5 checksum of a file-like object without reading its- whole content in memory.-- >>> from io import BytesIO- >>> md5sum(BytesIO(b'file content to hash'))- '784406af91dd5a54fbb9c84c2236595a'- """+ whole content in memory."""warnings.warn(("The scrapy.utils.misc.md5sum function is deprecated and will be "diff --git a/tests/ignores.txt b/tests/ignores.txtindex 94edcf186..3717bbc95 100644--- a/tests/ignores.txt+++ b/tests/ignores.txt@@ -1 +1,3 @@+scrapy/core/downloader/handlers/http.pyscrapy/extensions/statsmailer.py+scrapy/mail.pydiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 85d98f847..66b15bfdd 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -715,6 +715,9 @@ class TestCrawlSpider:assert isinstance(crawler.spider, SingleRequestSpider)assert crawler.spider.meta["responses"][0].certificate is None+ @pytest.mark.filterwarnings(+ r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"+ )@pytest.mark.parametrize("url",[diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 2ae9206a0..c4a8193c8 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -1148,6 +1148,9 @@ class TestHttpWithCrawlerBase(ABC):reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined]assert reason == "finished"+ @pytest.mark.filterwarnings(+ r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"+ )@coroutine_testasync def test_response_ssl_certificate(self, mockserver: MockServer) -> None:if not self.is_secure:diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.pyindex 28db389c3..4d208a1ea 100644--- a/tests/test_extension_statsmailer.py+++ b/tests/test_extension_statsmailer.py@@ -1,20 +1,27 @@+import warningsfrom unittest.mock import MagicMockimport pytestfrom scrapy import signals-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.signalmanager import SignalManagerfrom scrapy.statscollectors import StatsCollectorfrom scrapy.utils.spider import DefaultSpider-pytestmark = pytest.mark.filterwarnings(- "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",- "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",-)--from scrapy.extensions import statsmailer # noqa: E402-from scrapy.mail import MailSender # noqa: E402+with warnings.catch_warnings():+ warnings.filterwarnings(+ "ignore",+ r"The scrapy\.extensions\.statsmailer module is deprecated",+ ScrapyDeprecationWarning,+ )+ warnings.filterwarnings(+ "ignore",+ r"The scrapy\.mail module is deprecated",+ ScrapyDeprecationWarning,+ )+ from scrapy.extensions import statsmailer+ from scrapy.mail import MailSender@pytest.fixturediff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 28f306e31..3c1347fd3 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -669,6 +669,9 @@ class TestHttps2ClientProtocol:response = await make_request(client, request)assert response.status == status+ @pytest.mark.filterwarnings(+ r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"+ )@deferred_f_from_coro_fasync def test_response_has_correct_certificate_ip_address(self,diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex 57c209615..0af99ab6d 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport gzipimport re-import warningsfrom datetime import datetimefrom io import BytesIOfrom logging import WARNING@@ -429,9 +428,7 @@ Sitemap: /sitemap-relative-url.xmlcrawler = get_crawler(TestSpider)spider = TestSpider.from_crawler(crawler)- with warnings.catch_warnings():- warnings.simplefilter("error")- requests = [request async for request in spider.start()]+ requests = [request async for request in spider.start()]assert len(requests) == 1request = requests[0]diff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex aef2093ac..9257e0232 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -1,6 +1,5 @@from __future__ import annotations-import warningsfrom asyncio import sleepfrom typing import Any@@ -45,9 +44,7 @@ class TestMain:async def parse(self, response):yield ITEM_A- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_spider(TestSpider, [ITEM_A])+ await self._test_spider(TestSpider, [ITEM_A])@coroutine_testasync def test_start(self):@@ -57,9 +54,7 @@ class TestMain:async def start(self):yield ITEM_A- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_spider(TestSpider, [ITEM_A])+ await self._test_spider(TestSpider, [ITEM_A])@coroutine_testasync def test_start_subclass(self):@@ -70,9 +65,7 @@ class TestMain:class TestSpider(BaseSpider):name = "test"- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_spider(TestSpider, [ITEM_A])+ await self._test_spider(TestSpider, [ITEM_A])async def _test_start(self, start_, expected_items=None):class TestSpider(Spider):diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex 21df73a65..67ce3a920 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -1,4 +1,3 @@-import warningsfrom asyncio import sleepimport pytest@@ -76,9 +75,7 @@ class TestMain:@coroutine_testasync def test_modern_mw_modern_spider(self):- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)+ await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)async def _test_sleep(self, spider_middlewares):class TestSpider(Spider):diff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 20a3d940c..5333a55cb 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -1,5 +1,4 @@import os-import warningsfrom pathlib import Pathimport pytest@@ -41,11 +40,8 @@ class TestGetProjectSettings:envvars = {"SCRAPY_SETTINGS_MODULE": value,}- with warnings.catch_warnings():- warnings.simplefilter("error")- with set_environ(**envvars):- settings = get_project_settings()-+ with set_environ(**envvars):+ settings = get_project_settings()assert settings.get("SETTINGS_MODULE") == valuedef test_invalid_envvar(self):