Scrapy, a fast high-level web crawling & scraping framework for Python.
Assorted test improvements (#7232)
Andrey Rakhmatullin54d8562fb0f2
diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 9fd5a40df..1d64c2f39 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -633,43 +633,52 @@ class TestCrawlSpider:yield crawler.crawl(seed=url, mockserver=self.mockserver)assert crawler.spider.meta["responses"][0].certificate is None- @inlineCallbacks- def test_response_ssl_certificate(self):- crawler = get_crawler(SingleRequestSpider)- url = self.mockserver.url("/echo?body=test", is_secure=True)- yield crawler.crawl(seed=url, mockserver=self.mockserver)- cert = crawler.spider.meta["responses"][0].certificate- assert isinstance(cert, Certificate)- assert cert.getSubject().commonName == b"localhost"- assert cert.getIssuer().commonName == b"localhost"-- @pytest.mark.xfail(- reason="Responses with no body return early and contain no certificate"+ @pytest.mark.parametrize(+ "url",+ [+ "/echo?body=test",+ pytest.param(+ "/status?n=200",+ marks=pytest.mark.xfail(+ reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no certificate",+ strict=True,+ ),+ ),+ ],)- @inlineCallbacks- def test_response_ssl_certificate_empty_response(self):+ @deferred_f_from_coro_f+ async def test_response_ssl_certificate(+ self, mockserver: MockServer, url: str+ ) -> None:crawler = get_crawler(SingleRequestSpider)- url = self.mockserver.url("/status?n=200", is_secure=True)- yield crawler.crawl(seed=url, mockserver=self.mockserver)+ url = mockserver.url(url, is_secure=True)+ await crawler.crawl_async(seed=url, mockserver=mockserver)+ assert isinstance(crawler.spider, SingleRequestSpider)cert = crawler.spider.meta["responses"][0].certificateassert isinstance(cert, Certificate)assert cert.getSubject().commonName == b"localhost"assert cert.getIssuer().commonName == b"localhost"- @inlineCallbacks- def test_dns_server_ip_address_none(self):- crawler = get_crawler(SingleRequestSpider)- url = self.mockserver.url("/status?n=200")- yield crawler.crawl(seed=url, mockserver=self.mockserver)- ip_address = crawler.spider.meta["responses"][0].ip_address- assert ip_address is None-- @inlineCallbacks- def test_dns_server_ip_address(self):+ @pytest.mark.parametrize(+ "url",+ [+ "/echo?body=test",+ pytest.param(+ "/status?n=200",+ marks=pytest.mark.xfail(+ reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no ip_address",+ strict=True,+ ),+ ),+ ],+ )+ @deferred_f_from_coro_f+ async def test_response_ip_address(self, mockserver: MockServer, url: str) -> None:crawler = get_crawler(SingleRequestSpider)- url = self.mockserver.url("/echo?body=test")+ url = mockserver.url(url)expected_netloc, _ = urlparse(url).netloc.split(":")- yield crawler.crawl(seed=url, mockserver=self.mockserver)+ await crawler.crawl_async(seed=url, mockserver=mockserver)+ assert isinstance(crawler.spider, SingleRequestSpider)ip_address = crawler.spider.meta["responses"][0].ip_addressassert isinstance(ip_address, IPv4Address)assert str(ip_address) == gethostbyname(expected_netloc)diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.pyindex 79b2a6fc5..eb2735c4d 100644--- a/tests/test_downloader_handler_twisted_http11.py+++ b/tests/test_downloader_handler_twisted_http11.py@@ -60,7 +60,16 @@ class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCipherclass TestHttp11WithCrawler(TestHttpWithCrawlerBase):@propertydef settings_dict(self) -> dict[str, Any] | None:- return None # default handler settings+ return {+ "DOWNLOAD_HANDLERS": {+ "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",+ "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",+ }+ }+++class TestHttps11WithCrawler(TestHttp11WithCrawler):+ is_secure = Trueclass TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 3fbc8d416..5ae35a461 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -4,14 +4,12 @@ from __future__ import annotationsimport jsonfrom typing import TYPE_CHECKING, Any-from unittest import mockimport pytestfrom testfixtures import LogCapture-from twisted.internet import deferfrom twisted.web.http import H2_ENABLED-from scrapy.exceptions import DownloadCancelledError, UnsupportedURLSchemeError+from scrapy.exceptions import UnsupportedURLSchemeErrorfrom scrapy.http import Requestfrom scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom tests.test_downloader_handlers_http_base import (@@ -57,32 +55,6 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):response = await download_handler.download_request(request)assert response.protocol == "h2"- @deferred_f_from_coro_f- async def test_download_with_maxsize_very_large_file(- self, mockserver: MockServer- ) -> None:- from twisted.internet import reactor-- with mock.patch("scrapy.core.http2.stream.logger") as logger:- request = Request(- mockserver.url("/largechunkedfile", is_secure=self.is_secure)- )-- def check(logger: mock.Mock) -> None:- logger.error.assert_called_once_with(mock.ANY)-- async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:- with pytest.raises(DownloadCancelledError):- await download_handler.download_request(request)-- # As the error message is logged in the dataReceived callback, we- # have to give a bit of time to the reactor to process the queue- # after closing the connection.- d: defer.Deferred[mock.Mock] = defer.Deferred()- d.addCallback(check)- reactor.callLater(0.1, d.callback, logger)- await maybe_deferred_to_future(d)-def test_download_cause_data_loss(self) -> None: # type: ignore[override]pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)@@ -206,7 +178,8 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):def settings_dict(self) -> dict[str, Any] | None:return {"DOWNLOAD_HANDLERS": {- "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler"+ "http": None,+ "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",}}diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 5e3b49548..7ad787346 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -8,12 +8,13 @@ import sysfrom abc import ABC, abstractmethodfrom contextlib import asynccontextmanagerfrom http import HTTPStatus+from ipaddress import IPv4Address+from socket import gethostbynamefrom typing import TYPE_CHECKING, Any-from unittest import mock+from urllib.parse import urlparseimport pytest-from testfixtures import LogCapture-from twisted.internet import defer+from twisted.internet.ssl import Certificatefrom scrapy.exceptions import (CannotResolveHostError,@@ -25,7 +26,6 @@ from scrapy.exceptions import (UnsupportedURLSchemeError,)from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse-from scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import (deferred_f_from_coro_f,deferred_from_coro,@@ -135,6 +135,48 @@ class TestHttpBase(ABC):assert header_name in body["headers"]assert body["headers"][header_name] == [header_value]+ @deferred_f_from_coro_f+ async def test_request_header_none(self, mockserver: MockServer) -> None:+ """Adding a header with None as the value should not send that header."""+ request_headers = {+ "Cookie": None,+ "X-Custom-Header": None,+ }+ request = Request(+ mockserver.url("/echo", is_secure=self.is_secure),+ headers=request_headers,+ )+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.status == HTTPStatus.OK+ body = json.loads(response.body.decode("utf-8"))+ assert "headers" in body+ for header_name in request_headers:+ assert header_name not in body["headers"]++ @pytest.mark.parametrize(+ "request_headers",+ [+ {"X-Custom-Header": ["foo", "bar"]},+ [("X-Custom-Header", "foo"), ("X-Custom-Header", "bar")],+ ],+ )+ @deferred_f_from_coro_f+ async def test_request_header_duplicate(+ self, mockserver: MockServer, request_headers: Any+ ) -> None:+ """All values for a header should be sent."""+ request = Request(+ mockserver.url("/echo", is_secure=self.is_secure),+ headers=request_headers,+ )+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.status == HTTPStatus.OK+ body = json.loads(response.body.decode("utf-8"))+ assert "headers" in body+ assert body["headers"]["X-Custom-Header"] == ["foo", "bar"]+@deferred_f_from_coro_fasync def test_server_receives_correct_request_body(self, mockserver: MockServer@@ -166,7 +208,7 @@ class TestHttpBase(ABC):"Content-Encoding": "gzip","Content-MD5": "Q2hlY2sgSW50ZWdyaXR5IQ==","Content-Type": "text/html; charset=utf-8",- "Date": "Date: Tue, 15 Nov 1994 08:12:31 GMT",+ "Date": "Tue, 15 Nov 1994 08:12:31 GMT","Pragma": "no-cache","Retry-After": "120","Set-Cookie": "CookieName=CookieValue; Max-Age=3600; Version=1",@@ -450,28 +492,14 @@ class TestHttp11Base(TestHttpBase):@deferred_f_from_coro_fasync def test_download_with_maxsize_very_large_file(- self, mockserver: MockServer+ self, mockserver: MockServer, caplog: pytest.LogCaptureFixture) -> None:- # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11- with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:- request = Request(- mockserver.url("/largechunkedfile", is_secure=self.is_secure)- )-- def check(logger: mock.Mock) -> None:- logger.warning.assert_called_once_with(mock.ANY, mock.ANY)-- async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:- with pytest.raises(DownloadCancelledError):- await download_handler.download_request(request)+ request = Request(mockserver.url("/largechunkedfile", is_secure=self.is_secure))+ async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler:+ with pytest.raises(DownloadCancelledError):+ await download_handler.download_request(request)- # As the error message is logged in the dataReceived callback, we- # have to give a bit of time to the reactor to process the queue- # after closing the connection.- d: defer.Deferred[mock.Mock] = defer.Deferred()- d.addCallback(check)- call_later(0.1, d.callback, logger)- await maybe_deferred_to_future(d)+ assert "larger than download max size" in caplog.text@deferred_f_from_coro_fasync def test_download_with_maxsize_per_req(self, mockserver: MockServer) -> None:@@ -618,17 +646,17 @@ class TestHttps11Base(TestHttp11Base):pytest.skip("Unable to test on HTTPS")@deferred_f_from_coro_f- async def test_tls_logging(self, mockserver: MockServer) -> None:+ async def test_tls_logging(+ self, mockserver: MockServer, caplog: pytest.LogCaptureFixture+ ) -> None:request = Request(mockserver.url("/text", is_secure=self.is_secure))async with self.get_dh({"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}) as download_handler:- with LogCapture() as log_capture:+ with caplog.at_level("DEBUG"):response = await download_handler.download_request(request)assert response.body == b"Works"- log_capture.check_present(- ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message)- )+ assert self.tls_log_message in caplog.textclass TestSimpleHttpsBase(ABC):@@ -744,6 +772,33 @@ class TestHttpWithCrawlerBase(ABC):reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined]assert reason == "finished"+ @deferred_f_from_coro_f+ async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:+ if not self.is_secure:+ pytest.skip("Only applies to HTTPS")+ # copy of TestCrawl.test_response_ssl_certificate()+ # the current test implementation can only work for Twisted-based download handlers+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ url = mockserver.url("/echo?body=test", is_secure=self.is_secure)+ await crawler.crawl_async(seed=url, mockserver=mockserver)+ assert isinstance(crawler.spider, SingleRequestSpider)+ cert = crawler.spider.meta["responses"][0].certificate+ assert isinstance(cert, Certificate)+ assert cert.getSubject().commonName == b"localhost"+ assert cert.getIssuer().commonName == b"localhost"++ @deferred_f_from_coro_f+ async def test_response_ip_address(self, mockserver: MockServer) -> None:+ # copy of TestCrawl.test_response_ip_address()+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)+ url = mockserver.url("/echo?body=test", is_secure=self.is_secure)+ expected_netloc, _ = urlparse(url).netloc.split(":")+ await crawler.crawl_async(seed=url, mockserver=mockserver)+ assert isinstance(crawler.spider, SingleRequestSpider)+ ip_address = crawler.spider.meta["responses"][0].ip_address+ assert isinstance(ip_address, IPv4Address)+ assert str(ip_address) == gethostbyname(expected_netloc)+class TestHttpProxyBase(ABC):is_secure = Falsediff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.pyindex 1d7df70eb..b46ebe2d9 100644--- a/tests/test_engine_stop_download_bytes.py+++ b/tests/test_engine_stop_download_bytes.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from testfixtures import LogCapture-from scrapy.exceptions import StopDownloadfrom scrapy.utils.defer import deferred_f_from_coro_ffrom tests.test_engine import (@@ -16,6 +14,8 @@ from tests.test_engine import ()if TYPE_CHECKING:+ import pytest+from tests.mockserver.http import MockServer@@ -27,7 +27,9 @@ class BytesReceivedCrawlerRun(CrawlerRun):class TestBytesReceivedEngine(TestEngineBase):@deferred_f_from_coro_f- async def test_crawler(self, mockserver: MockServer) -> None:+ async def test_crawler(+ self, mockserver: MockServer, caplog: pytest.LogCaptureFixture+ ) -> None:for spider in (MySpider,DictItemsSpider,@@ -35,32 +37,13 @@ class TestBytesReceivedEngine(TestEngineBase):DataClassItemsSpider,):run = BytesReceivedCrawlerRun(spider)- with LogCapture() as log:+ with caplog.at_level("DEBUG"):await run.run(mockserver)- log.check_present(- (- "scrapy.core.downloader.handlers.http11",- "DEBUG",- f"Download stopped for <GET {mockserver.url('/redirected')}> "- "from signal handler BytesReceivedCrawlerRun.bytes_received",- )- )- log.check_present(- (- "scrapy.core.downloader.handlers.http11",- "DEBUG",- f"Download stopped for <GET {mockserver.url('/static/')}> "- "from signal handler BytesReceivedCrawlerRun.bytes_received",- )- )- log.check_present(- (- "scrapy.core.downloader.handlers.http11",- "DEBUG",- f"Download stopped for <GET {mockserver.url('/numbers')}> "- "from signal handler BytesReceivedCrawlerRun.bytes_received",- )- )+ for url in ("/redirected", "/static/", "/numbers"):+ assert (+ f"Download stopped for <GET {mockserver.url(url)}> "+ "from signal handler BytesReceivedCrawlerRun.bytes_received"+ ) in caplog.textself._assert_visited_urls(run)self._assert_scheduled_requests(run, count=9)self._assert_downloaded_responses(run, count=9)diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.pyindex c01413d4e..76cfbe8ed 100644--- a/tests/test_engine_stop_download_headers.py+++ b/tests/test_engine_stop_download_headers.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from testfixtures import LogCapture-from scrapy.exceptions import StopDownloadfrom scrapy.utils.defer import deferred_f_from_coro_ffrom tests.test_engine import (@@ -16,6 +14,8 @@ from tests.test_engine import ()if TYPE_CHECKING:+ import pytest+from tests.mockserver.http import MockServer@@ -27,7 +27,9 @@ class HeadersReceivedCrawlerRun(CrawlerRun):class TestHeadersReceivedEngine(TestEngineBase):@deferred_f_from_coro_f- async def test_crawler(self, mockserver: MockServer) -> None:+ async def test_crawler(+ self, mockserver: MockServer, caplog: pytest.LogCaptureFixture+ ) -> None:for spider in (MySpider,DictItemsSpider,@@ -35,32 +37,13 @@ class TestHeadersReceivedEngine(TestEngineBase):DataClassItemsSpider,):run = HeadersReceivedCrawlerRun(spider)- with LogCapture() as log:+ with caplog.at_level("DEBUG"):await run.run(mockserver)- log.check_present(- (- "scrapy.core.downloader.handlers.http11",- "DEBUG",- f"Download stopped for <GET {mockserver.url('/redirected')}> from"- " signal handler HeadersReceivedCrawlerRun.headers_received",- )- )- log.check_present(- (- "scrapy.core.downloader.handlers.http11",- "DEBUG",- f"Download stopped for <GET {mockserver.url('/static/')}> from signal"- " handler HeadersReceivedCrawlerRun.headers_received",- )- )- log.check_present(- (- "scrapy.core.downloader.handlers.http11",- "DEBUG",- f"Download stopped for <GET {mockserver.url('/numbers')}> from"- " signal handler HeadersReceivedCrawlerRun.headers_received",- )- )+ for url in ("/redirected", "/static/", "/numbers"):+ assert (+ f"Download stopped for <GET {mockserver.url(url)}> "+ "from signal handler HeadersReceivedCrawlerRun.headers_received"+ ) in caplog.textself._assert_visited_urls(run)self._assert_downloaded_responses(run, count=6)self._assert_signals_caught(run)diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex f315388cd..acf8dd412 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -476,7 +476,8 @@ class TestMiddlewareManagerSpider:):await mwman.close_spider_async()- def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:+ @deferred_f_from_coro_f+ async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:"""Crawler is provided and has a spider, works. The instance passed to a deprecated methodis ignored, even if mismatched."""mwman = ItemPipelineManager(crawler=crawler)@@ -485,14 +486,15 @@ class TestMiddlewareManagerSpider:ScrapyDeprecationWarning,match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead",):- mwman.open_spider(DefaultSpider())+ await maybe_deferred_to_future(mwman.open_spider(DefaultSpider()))with pytest.warns(ScrapyDeprecationWarning,match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead",):- mwman.close_spider(DefaultSpider())+ await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))- def test_deprecated_spider_arg_without_crawler(self) -> None:+ @deferred_f_from_coro_f+ async def test_deprecated_spider_arg_without_crawler(self) -> None:"""The first instance passed to a deprecated method is used. Mismatched ones raise an error."""with pytest.warns(ScrapyDeprecationWarning,@@ -504,7 +506,7 @@ class TestMiddlewareManagerSpider:ScrapyDeprecationWarning,match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead",):- mwman.open_spider(spider)+ await maybe_deferred_to_future(mwman.open_spider(spider))with (pytest.warns(ScrapyDeprecationWarning,@@ -514,12 +516,12 @@ class TestMiddlewareManagerSpider:RuntimeError, match="Different instances of Spider were passed"),):- mwman.close_spider(DefaultSpider())+ await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))with pytest.warns(ScrapyDeprecationWarning,match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead",):- mwman.close_spider(spider)+ await maybe_deferred_to_future(mwman.close_spider(spider))@deferred_f_from_coro_fasync def test_no_spider_arg_without_crawler(self) -> None:diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex df7f1a20e..be623c3fb 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -321,9 +321,11 @@ class TestDeferredFFromCoroF:yield self._assert_result(c_f)+ @pytest.mark.only_asyncio@inlineCallbacksdef test_coroutine_asyncio(self):async def c_f() -> int:+ await asyncio.sleep(0.01)return 42yield self._assert_result(c_f)