Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Address warnings in tests. (#7637)

Andrey Rakhmatullin3f3cb885eddf
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index c0da7dfa4..ebbdf1b98 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -51,6 +51,8 @@ class Contract:
results.addSuccess(self.testcase_pre)
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+ if isinstance(cb_result, CoroutineType):
+ cb_result.close()
raise TypeError("Contracts don't support async callbacks")
return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
@@ -67,6 +69,8 @@ class Contract:
def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+ if isinstance(cb_result, CoroutineType):
+ cb_result.close()
raise TypeError("Contracts don't support async callbacks")
output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
try:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 47568e656..20e7cb381 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover
return list(walk_modules_iter(path))
-def md5sum(file: IO[bytes]) -> str:
+def md5sum(file: IO[bytes]) -> str: # pragma: no cover
"""Calculate the md5 checksum of a file-like object without reading its
- whole content in memory.
-
- >>> from io import BytesIO
- >>> md5sum(BytesIO(b'file content to hash'))
- '784406af91dd5a54fbb9c84c2236595a'
- """
+ whole content in memory."""
warnings.warn(
(
"The scrapy.utils.misc.md5sum function is deprecated and will be "
diff --git a/tests/ignores.txt b/tests/ignores.txt
index 94edcf186..3717bbc95 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1 +1,3 @@
+scrapy/core/downloader/handlers/http.py
scrapy/extensions/statsmailer.py
+scrapy/mail.py
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 85d98f847..66b15bfdd 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -715,6 +715,9 @@ class TestCrawlSpider:
assert isinstance(crawler.spider, SingleRequestSpider)
assert crawler.spider.meta["responses"][0].certificate is None
+ @pytest.mark.filterwarnings(
+ r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
+ )
@pytest.mark.parametrize(
"url",
[
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 2ae9206a0..c4a8193c8 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -1148,6 +1148,9 @@ class TestHttpWithCrawlerBase(ABC):
reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined]
assert reason == "finished"
+ @pytest.mark.filterwarnings(
+ r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
+ )
@coroutine_test
async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
if not self.is_secure:
diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py
index 28db389c3..4d208a1ea 100644
--- a/tests/test_extension_statsmailer.py
+++ b/tests/test_extension_statsmailer.py
@@ -1,20 +1,27 @@
+import warnings
from unittest.mock import MagicMock
import pytest
from scrapy import signals
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.spider import DefaultSpider
-pytestmark = pytest.mark.filterwarnings(
- "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",
- "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",
-)
-
-from scrapy.extensions import statsmailer # noqa: E402
-from scrapy.mail import MailSender # noqa: E402
+with warnings.catch_warnings():
+ warnings.filterwarnings(
+ "ignore",
+ r"The scrapy\.extensions\.statsmailer module is deprecated",
+ ScrapyDeprecationWarning,
+ )
+ warnings.filterwarnings(
+ "ignore",
+ r"The scrapy\.mail module is deprecated",
+ ScrapyDeprecationWarning,
+ )
+ from scrapy.extensions import statsmailer
+ from scrapy.mail import MailSender
@pytest.fixture
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 28f306e31..3c1347fd3 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -669,6 +669,9 @@ class TestHttps2ClientProtocol:
response = await make_request(client, request)
assert response.status == status
+ @pytest.mark.filterwarnings(
+ r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
+ )
@deferred_f_from_coro_f
async def test_response_has_correct_certificate_ip_address(
self,
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index 57c209615..0af99ab6d 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import gzip
import re
-import warnings
from datetime import datetime
from io import BytesIO
from logging import WARNING
@@ -429,9 +428,7 @@ Sitemap: /sitemap-relative-url.xml
crawler = get_crawler(TestSpider)
spider = TestSpider.from_crawler(crawler)
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- requests = [request async for request in spider.start()]
+ requests = [request async for request in spider.start()]
assert len(requests) == 1
request = requests[0]
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index aef2093ac..9257e0232 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import warnings
from asyncio import sleep
from typing import Any
@@ -45,9 +44,7 @@ class TestMain:
async def parse(self, response):
yield ITEM_A
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_spider(TestSpider, [ITEM_A])
+ await self._test_spider(TestSpider, [ITEM_A])
@coroutine_test
async def test_start(self):
@@ -57,9 +54,7 @@ class TestMain:
async def start(self):
yield ITEM_A
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_spider(TestSpider, [ITEM_A])
+ await self._test_spider(TestSpider, [ITEM_A])
@coroutine_test
async def test_start_subclass(self):
@@ -70,9 +65,7 @@ class TestMain:
class TestSpider(BaseSpider):
name = "test"
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_spider(TestSpider, [ITEM_A])
+ await self._test_spider(TestSpider, [ITEM_A])
async def _test_start(self, start_, expected_items=None):
class TestSpider(Spider):
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index 21df73a65..67ce3a920 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -1,4 +1,3 @@
-import warnings
from asyncio import sleep
import pytest
@@ -76,9 +75,7 @@ class TestMain:
@coroutine_test
async def test_modern_mw_modern_spider(self):
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
+ await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
async def _test_sleep(self, spider_middlewares):
class TestSpider(Spider):
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 20a3d940c..5333a55cb 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,5 +1,4 @@
import os
-import warnings
from pathlib import Path
import pytest
@@ -41,11 +40,8 @@ class TestGetProjectSettings:
envvars = {
"SCRAPY_SETTINGS_MODULE": value,
}
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- with set_environ(**envvars):
- settings = get_project_settings()
-
+ with set_environ(**envvars):
+ settings = get_project_settings()
assert settings.get("SETTINGS_MODULE") == value
def test_invalid_envvar(self):