Scrapy, a fast high-level web crawling & scraping framework for Python.
Rewrite download handler tests to coroutines. (#6846)
Andrey Rakhmatullin8ae418df44b7
diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.pyindex 807c8c4cb..bc306aa07 100644--- a/tests/test_downloader_handler_twisted_http10.py+++ b/tests/test_downloader_handler_twisted_http10.py@@ -9,6 +9,7 @@ import pytestfrom scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandlerfrom scrapy.http import Requestfrom scrapy.spiders import Spider+from scrapy.utils.defer import deferred_f_from_coro_ffrom tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBaseif TYPE_CHECKING:@@ -25,12 +26,11 @@ class HTTP10DownloadHandlerMixin:class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase):"""HTTP 1.0 test case"""- def test_protocol(self):+ @deferred_f_from_coro_f+ async def test_protocol(self):request = Request(self.getURL("host"), method="GET")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.protocol)- d.addCallback(self.assertEqual, "HTTP/1.0")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.protocol == "HTTP/1.0"class TestHttps10(TestHttp10):diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 159f403d0..e058cedae 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -15,6 +15,10 @@ from twisted.web.http import H2_ENABLEDfrom scrapy.http import Requestfrom scrapy.spiders import Spider+from scrapy.utils.defer import (+ deferred_f_from_coro_f,+ maybe_deferred_to_future,+)from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import ssl_context_factory@@ -50,15 +54,14 @@ class H2DownloadHandlerMixin:class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"- def test_protocol(self):+ @deferred_f_from_coro_f+ async def test_protocol(self):request = Request(self.getURL("host"), method="GET")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.protocol)- d.addCallback(self.assertEqual, "h2")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.protocol == "h2"- @defer.inlineCallbacks- def test_download_with_maxsize_very_large_file(self):+ @deferred_f_from_coro_f+ async def test_download_with_maxsize_very_large_file(self):from twisted.internet import reactorwith mock.patch("scrapy.core.http2.stream.logger") as logger:@@ -67,8 +70,10 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):def check(logger):logger.error.assert_called_once_with(mock.ANY)- d = self.download_request(request, Spider("foo", download_maxsize=1500))- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await self.download_request(+ request, Spider("foo", download_maxsize=1500)+ )# As the error message is logged in the dataReceived callback, we# have to give a bit of time to the reactor to process the queue@@ -76,13 +81,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):d = defer.Deferred()d.addCallback(check)reactor.callLater(0.1, d.callback, logger)- yield d+ await maybe_deferred_to_future(d)- @defer.inlineCallbacks- def test_unsupported_scheme(self):+ @deferred_f_from_coro_f+ async def test_unsupported_scheme(self):request = Request("ftp://unsupported.scheme")- d = self.download_request(request, Spider("foo"))- yield self.assertFailure(d, SchemeNotSupported)+ with pytest.raises(SchemeNotSupported):+ await self.download_request(request, Spider("foo"))def test_download_broken_content_cause_data_loss(self, url="broken"):pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)@@ -102,70 +107,60 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):def test_download_broken_chunked_content_allow_data_loss_via_setting(self):pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)- def test_concurrent_requests_same_domain(self):+ @deferred_f_from_coro_f+ async def test_concurrent_requests_same_domain(self):spider = Spider("foo")request1 = Request(self.getURL("file"))- d1 = self.download_request(request1, spider)- d1.addCallback(lambda r: r.body)- d1.addCallback(self.assertEqual, b"0123456789")+ response1 = await self.download_request(request1, spider)+ assert response1.body == b"0123456789"request2 = Request(self.getURL("echo"), method="POST")- d2 = self.download_request(request2, spider)- d2.addCallback(lambda r: r.headers["Content-Length"])- d2.addCallback(self.assertEqual, b"79")-- return defer.DeferredList([d1, d2])+ response2 = await self.download_request(request2, spider)+ assert response2.headers["Content-Length"] == b"79"@pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")- def test_connect_request(self):+ @deferred_f_from_coro_f+ async def test_connect_request(self):request = Request(self.getURL("file"), method="CONNECT")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b""- def test_custom_content_length_good(self):+ @deferred_f_from_coro_f+ async def test_custom_content_length_good(self):request = Request(self.getURL("contentlength"))custom_content_length = str(len(request.body))request.headers["Content-Length"] = custom_content_length- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.text)- d.addCallback(self.assertEqual, custom_content_length)- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.text == custom_content_length- def test_custom_content_length_bad(self):+ @deferred_f_from_coro_f+ async def test_custom_content_length_bad(self):request = Request(self.getURL("contentlength"))actual_content_length = str(len(request.body))bad_content_length = str(len(request.body) + 1)request.headers["Content-Length"] = bad_content_length- log = LogCapture()- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.text)- d.addCallback(self.assertEqual, actual_content_length)- d.addCallback(- lambda _: log.check_present(- (- "scrapy.core.http2.stream",- "WARNING",- f"Ignoring bad Content-Length header "- f"{bad_content_length!r} of request {request}, sending "- f"{actual_content_length!r} instead",- )+ with LogCapture() as log:+ response = await self.download_request(request, Spider("foo"))+ assert response.text == actual_content_length+ log.check_present(+ (+ "scrapy.core.http2.stream",+ "WARNING",+ f"Ignoring bad Content-Length header "+ f"{bad_content_length!r} of request {request}, sending "+ f"{actual_content_length!r} instead",))- d.addCallback(lambda _: log.uninstall())- return d- def test_duplicate_header(self):+ @deferred_f_from_coro_f+ async def test_duplicate_header(self):request = Request(self.getURL("echo"))header, value1, value2 = "Custom-Header", "foo", "bar"request.headers.appendlist(header, value1)request.headers.appendlist(header, value2)- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: json.loads(r.text)["headers"][header])- d.addCallback(self.assertEqual, [value1, value2])- return d+ response = await self.download_request(request, Spider("foo"))+ assert json.loads(response.text)["headers"][header] == [value1, value2]class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):@@ -222,12 +217,13 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase):self.download_handler = build_from_crawler(self.download_handler_cls, get_crawler())- self.download_request = self.download_handler.download_requestdef getURL(self, path):return f"{self.scheme}://{self.host}:{self.portno}/{path}"- @defer.inlineCallbacks- def test_download_with_proxy_https_timeout(self):+ @deferred_f_from_coro_f+ async def test_download_with_proxy_https_timeout(self):with pytest.raises(NotImplementedError):- yield super().test_download_with_proxy_https_timeout()+ await maybe_deferred_to_future(+ super().test_download_with_proxy_https_timeout()+ )diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex dacadb075..09cdbaf35 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -22,10 +22,14 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.ftp import FTPDownloadHandlerfrom scrapy.core.downloader.handlers.s3 import S3DownloadHandlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import HtmlResponse, Request+from scrapy.http import HtmlResponse, Request, Responsefrom scrapy.http.response.text import TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider+from scrapy.utils.defer import (+ deferred_f_from_coro_f,+ maybe_deferred_to_future,+)from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler@@ -95,28 +99,33 @@ class TestFile(unittest.TestCase):# add a special char to check that they are handled correctlyself.fd, self.tmpname = mkstemp(suffix="^")Path(self.tmpname).write_text("0123456789", encoding="utf-8")- handler = build_from_crawler(FileDownloadHandler, get_crawler())- self.download_request = handler.download_request+ self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler())def tearDown(self):os.close(self.fd)Path(self.tmpname).unlink()- def test_download(self):- def _test(response):- assert response.url == request.url- assert response.status == 200- assert response.body == b"0123456789"- assert response.protocol is None+ async def download_request(self, request: Request, spider: Spider) -> Response:+ return await maybe_deferred_to_future(+ self.download_handler.download_request(request, spider)+ )+ @deferred_f_from_coro_f+ async def test_download(self):request = Request(path_to_file_uri(self.tmpname))assert request.url.upper().endswith("%5E")- return self.download_request(request, Spider("foo")).addCallback(_test)-- def test_non_existent(self):+ response = await self.download_request(request, Spider("foo"))+ assert response.url == request.url+ assert response.status == 200+ assert response.body == b"0123456789"+ assert response.protocol is None++ @deferred_f_from_coro_f+ async def test_non_existent(self):request = Request(path_to_file_uri(mkdtemp()))- d = self.download_request(request, Spider("foo"))- return self.assertFailure(d, OSError)+ # the specific exception differs between platforms+ with pytest.raises(OSError): # noqa: PT011+ await self.download_request(request, Spider("foo"))class HttpDownloadHandlerMock:@@ -479,69 +488,65 @@ class TestDataURI(unittest.TestCase):def setUp(self):crawler = get_crawler()self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler)- self.download_request = self.download_handler.download_requestself.spider = Spider("foo")- def test_response_attrs(self):- uri = "data:,A%20brief%20note"-- def _test(response):- assert response.url == uri- assert not response.headers+ async def download_request(self, request: Request, spider: Spider) -> Response:+ return await maybe_deferred_to_future(+ self.download_handler.download_request(request, spider)+ )+ @deferred_f_from_coro_f+ async def test_response_attrs(self):+ uri = "data:,A%20brief%20note"request = Request(uri)- return self.download_request(request, self.spider).addCallback(_test)-- def test_default_mediatype_encoding(self):- def _test(response):- assert response.text == "A brief note"- assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck- assert response.encoding == "US-ASCII"+ response = await self.download_request(request, self.spider)+ assert response.url == uri+ assert not response.headers+ @deferred_f_from_coro_f+ async def test_default_mediatype_encoding(self):request = Request("data:,A%20brief%20note")- return self.download_request(request, self.spider).addCallback(_test)-- def test_default_mediatype(self):- def _test(response):- assert response.text == "\u038e\u03a3\u038e"- assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck- assert response.encoding == "iso-8859-7"+ response = await self.download_request(request, self.spider)+ assert response.text == "A brief note"+ assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck+ assert response.encoding == "US-ASCII"+ @deferred_f_from_coro_f+ async def test_default_mediatype(self):request = Request("data:;charset=iso-8859-7,%be%d3%be")- return self.download_request(request, self.spider).addCallback(_test)-- def test_text_charset(self):- def _test(response):- assert response.text == "\u038e\u03a3\u038e"- assert response.body == b"\xbe\xd3\xbe"- assert response.encoding == "iso-8859-7"+ response = await self.download_request(request, self.spider)+ assert response.text == "\u038e\u03a3\u038e"+ assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck+ assert response.encoding == "iso-8859-7"+ @deferred_f_from_coro_f+ async def test_text_charset(self):request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")- return self.download_request(request, self.spider).addCallback(_test)-- def test_mediatype_parameters(self):- def _test(response):- assert response.text == "\u038e\u03a3\u038e"- assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck- assert response.encoding == "utf-8"+ response = await self.download_request(request, self.spider)+ assert response.text == "\u038e\u03a3\u038e"+ assert response.body == b"\xbe\xd3\xbe"+ assert response.encoding == "iso-8859-7"+ @deferred_f_from_coro_f+ async def test_mediatype_parameters(self):request = Request("data:text/plain;foo=%22foo;bar%5C%22%22;""charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22"",%CE%8E%CE%A3%CE%8E")- return self.download_request(request, self.spider).addCallback(_test)-- def test_base64(self):- def _test(response):- assert response.text == "Hello, world."+ response = await self.download_request(request, self.spider)+ assert response.text == "\u038e\u03a3\u038e"+ assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck+ assert response.encoding == "utf-8"+ @deferred_f_from_coro_f+ async def test_base64(self):request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D")- return self.download_request(request, self.spider).addCallback(_test)-- def test_protocol(self):- def _test(response):- assert response.protocol is None+ response = await self.download_request(request, self.spider)+ assert response.text == "Hello, world."+ @deferred_f_from_coro_f+ async def test_protocol(self):request = Request("data:,")- return self.download_request(request, self.spider).addCallback(_test)+ response = await self.download_request(request, self.spider)+ assert response.protocol is Nonediff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 5eaf66966..14e12a3e6 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -2,6 +2,7 @@from __future__ import annotations+import jsonimport shutilimport sysfrom abc import ABC, abstractmethod@@ -13,14 +14,20 @@ from unittest import mockimport pytestfrom testfixtures import LogCapturefrom twisted.internet import defer, error+from twisted.internet.defer import maybeDeferredfrom twisted.protocols.policies import WrappingFactoryfrom twisted.trial import unittestfrom twisted.web import resource, server, static, utilfrom twisted.web._newclient import ResponseFailedfrom twisted.web.http import _DataLoss-from scrapy.http import Headers, HtmlResponse, Request, TextResponse+from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponsefrom scrapy.spiders import Spider+from scrapy.utils.defer import (+ deferred_f_from_coro_f,+ deferred_from_coro,+ maybe_deferred_to_future,+)from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler@@ -178,7 +185,6 @@ class TestHttpBase(unittest.TestCase, ABC):self.download_handler = build_from_crawler(self.download_handler_cls, get_crawler())- self.download_request = self.download_handler.download_request@defer.inlineCallbacksdef tearDown(self):@@ -190,36 +196,37 @@ class TestHttpBase(unittest.TestCase, ABC):def getURL(self, path):return f"{self.scheme}://{self.host}:{self.portno}/{path}"- def test_download(self):+ async def download_request(self, request: Request, spider: Spider) -> Response:+ return await maybe_deferred_to_future(+ self.download_handler.download_request(request, spider)+ )++ @deferred_f_from_coro_f+ async def test_download(self):request = Request(self.getURL("file"))- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"0123456789")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b"0123456789"- def test_download_head(self):+ @deferred_f_from_coro_f+ async def test_download_head(self):request = Request(self.getURL("file"), method="HEAD")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b""- def test_redirect_status(self):+ @deferred_f_from_coro_f+ async def test_redirect_status(self):request = Request(self.getURL("redirect"))- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.status)- d.addCallback(self.assertEqual, 302)- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.status == 302- def test_redirect_status_head(self):+ @deferred_f_from_coro_f+ async def test_redirect_status_head(self):request = Request(self.getURL("redirect"), method="HEAD")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.status)- d.addCallback(self.assertEqual, 302)- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.status == 302- @defer.inlineCallbacks- def test_timeout_download_from_spider_nodata_rcvd(self):+ @deferred_f_from_coro_f+ async def test_timeout_download_from_spider_nodata_rcvd(self):if self.reactor_pytest != "default" and sys.platform == "win32":# https://twistedmatrix.com/trac/ticket/10279raise unittest.SkipTest(@@ -230,11 +237,12 @@ class TestHttpBase(unittest.TestCase, ABC):spider = Spider("foo")meta = {"download_timeout": 0.5}request = Request(self.getURL("wait"), meta=meta)- d = self.download_request(request, spider)- yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)+ d = deferred_from_coro(self.download_request(request, spider))+ with pytest.raises((defer.TimeoutError, error.TimeoutError)):+ await maybe_deferred_to_future(d)- @defer.inlineCallbacks- def test_timeout_download_from_spider_server_hangs(self):+ @deferred_f_from_coro_f+ async def test_timeout_download_from_spider_server_hangs(self):if self.reactor_pytest != "default" and sys.platform == "win32":# https://twistedmatrix.com/trac/ticket/10279raise unittest.SkipTest(@@ -244,28 +252,27 @@ class TestHttpBase(unittest.TestCase, ABC):spider = Spider("foo")meta = {"download_timeout": 0.5}request = Request(self.getURL("hang-after-headers"), meta=meta)- d = self.download_request(request, spider)- yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)-- def test_host_header_not_in_request_headers(self):- def _test(response):- assert response.body == to_bytes(f"{self.host}:{self.portno}")- assert not request.headers+ d = deferred_from_coro(self.download_request(request, spider))+ with pytest.raises((defer.TimeoutError, error.TimeoutError)):+ await maybe_deferred_to_future(d)+ @deferred_f_from_coro_f+ async def test_host_header_not_in_request_headers(self):request = Request(self.getURL("host"))- return self.download_request(request, Spider("foo")).addCallback(_test)+ response = await self.download_request(request, Spider("foo"))+ assert response.body == to_bytes(f"{self.host}:{self.portno}")+ assert not request.headers- def test_host_header_seted_in_request_headers(self):+ @deferred_f_from_coro_f+ async def test_host_header_set_in_request_headers(self):host = self.host + ":" + str(self.portno)-- def _test(response):- assert response.body == host.encode()- assert request.headers.get("Host") == host.encode()-request = Request(self.getURL("host"), headers={"Host": host})- return self.download_request(request, Spider("foo")).addCallback(_test)+ response = await self.download_request(request, Spider("foo"))+ assert response.body == host.encode()+ assert request.headers.get("Host") == host.encode()- def test_content_length_zero_bodyless_post_request_headers(self):+ @deferred_f_from_coro_f+ async def test_content_length_zero_bodyless_post_request_headers(self):"""Tests if "Content-Length: 0" is sent for bodyless POST requests.This is not strictly required by HTTP RFCs but can cause trouble@@ -276,105 +283,93 @@ class TestHttpBase(unittest.TestCase, ABC):https://github.com/kennethreitz/requests/issues/405https://bugs.python.org/issue14721"""-- def _test(response):- assert response.body == b"0"-request = Request(self.getURL("contentlength"), method="POST")- return self.download_request(request, Spider("foo")).addCallback(_test)-- def test_content_length_zero_bodyless_post_only_one(self):- def _test(response):- import json-- headers = Headers(json.loads(response.text)["headers"])- contentlengths = headers.getlist("Content-Length")- assert len(contentlengths) == 1- assert contentlengths == [b"0"]+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b"0"+ @deferred_f_from_coro_f+ async def test_content_length_zero_bodyless_post_only_one(self):request = Request(self.getURL("echo"), method="POST")- return self.download_request(request, Spider("foo")).addCallback(_test)-- def test_payload(self):+ response = await self.download_request(request, Spider("foo"))+ headers = Headers(json.loads(response.text)["headers"])+ contentlengths = headers.getlist("Content-Length")+ assert len(contentlengths) == 1+ assert contentlengths == [b"0"]++ @deferred_f_from_coro_f+ async def test_payload(self):body = b"1" * 100 # PayloadResource requires body length to be 100request = Request(self.getURL("payload"), method="POST", body=body)- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, body)- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == body- def test_response_header_content_length(self):+ @deferred_f_from_coro_f+ async def test_response_header_content_length(self):request = Request(self.getURL("file"), method=b"GET")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.headers[b"content-length"])- d.addCallback(self.assertEqual, b"159")- return d-- def _test_response_class(self, filename, body, response_class):- def _test(response):- assert type(response) is response_class # pylint: disable=unidiomatic-typecheck+ response = await self.download_request(request, Spider("foo"))+ assert response.headers[b"content-length"] == b"159"+ async def _test_response_class(+ self, filename: str, body: bytes, response_class: type[Response]+ ) -> None:request = Request(self.getURL(filename), body=body)- return self.download_request(request, Spider("foo")).addCallback(_test)+ response = await self.download_request(request, Spider("foo"))+ assert type(response) is response_class # pylint: disable=unidiomatic-typecheck- def test_response_class_from_url(self):- return self._test_response_class("foo.html", b"", HtmlResponse)+ @deferred_f_from_coro_f+ async def test_response_class_from_url(self):+ await self._test_response_class("foo.html", b"", HtmlResponse)- def test_response_class_from_body(self):- return self._test_response_class(+ @deferred_f_from_coro_f+ async def test_response_class_from_body(self):+ await self._test_response_class("foo",b"<!DOCTYPE html>\n<title>.</title>",HtmlResponse,)- def test_get_duplicate_header(self):- def _test(response):- assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]-+ @deferred_f_from_coro_f+ async def test_get_duplicate_header(self):request = Request(self.getURL("duplicate-header"))- return self.download_request(request, Spider("foo")).addCallback(_test)+ response = await self.download_request(request, Spider("foo"))+ assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]class TestHttp11Base(TestHttpBase):"""HTTP 1.1 test case"""- def test_download_without_maxsize_limit(self):+ @deferred_f_from_coro_f+ async def test_download_without_maxsize_limit(self):request = Request(self.getURL("file"))- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"0123456789")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b"0123456789"- def test_response_class_choosing_request(self):+ @deferred_f_from_coro_f+ async def test_response_class_choosing_request(self):"""Tests choosing of correct response typein case of Content-Type is empty but body contains text."""body = b"Some plain text\ndata with tabs\t and null bytes\0"-- def _test_type(response):- assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck-request = Request(self.getURL("nocontenttype"), body=body)- d = self.download_request(request, Spider("foo"))- d.addCallback(_test_type)- return d+ response = await self.download_request(request, Spider("foo"))+ assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck- @defer.inlineCallbacks- def test_download_with_maxsize(self):+ @deferred_f_from_coro_f+ async def test_download_with_maxsize(self):request = Request(self.getURL("file"))# 10 is minimal size for this request and the limit is only counted on# response body. (regardless of headers)- d = self.download_request(request, Spider("foo", download_maxsize=10))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"0123456789")- yield d+ response = await self.download_request(+ request, Spider("foo", download_maxsize=10)+ )+ assert response.body == b"0123456789"- d = self.download_request(request, Spider("foo", download_maxsize=9))- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await self.download_request(request, Spider("foo", download_maxsize=9))- @defer.inlineCallbacks- def test_download_with_maxsize_very_large_file(self):+ @deferred_f_from_coro_f+ async def test_download_with_maxsize_very_large_file(self):from twisted.internet import reactor# TODO: the logger check is specific to scrapy.core.downloader.handlers.http11@@ -384,8 +379,10 @@ class TestHttp11Base(TestHttpBase):def check(logger):logger.warning.assert_called_once_with(mock.ANY, mock.ANY)- d = self.download_request(request, Spider("foo", download_maxsize=1500))- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await self.download_request(+ request, Spider("foo", download_maxsize=1500)+ )# As the error message is logged in the dataReceived callback, we# have to give a bit of time to the reactor to process the queue@@ -393,84 +390,81 @@ class TestHttp11Base(TestHttpBase):d = defer.Deferred()d.addCallback(check)reactor.callLater(0.1, d.callback, logger)- yield d+ await maybe_deferred_to_future(d)- @defer.inlineCallbacks- def test_download_with_maxsize_per_req(self):+ @deferred_f_from_coro_f+ async def test_download_with_maxsize_per_req(self):meta = {"download_maxsize": 2}request = Request(self.getURL("file"), meta=meta)- d = self.download_request(request, Spider("foo"))- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await self.download_request(request, Spider("foo"))- @defer.inlineCallbacks- def test_download_with_small_maxsize_per_spider(self):+ @deferred_f_from_coro_f+ async def test_download_with_small_maxsize_per_spider(self):request = Request(self.getURL("file"))- d = self.download_request(request, Spider("foo", download_maxsize=2))- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):+ await self.download_request(request, Spider("foo", download_maxsize=2))- def test_download_with_large_maxsize_per_spider(self):+ @deferred_f_from_coro_f+ async def test_download_with_large_maxsize_per_spider(self):request = Request(self.getURL("file"))- d = self.download_request(request, Spider("foo", download_maxsize=100))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"0123456789")- return d+ response = await self.download_request(+ request, Spider("foo", download_maxsize=100)+ )+ assert response.body == b"0123456789"- def test_download_chunked_content(self):+ @deferred_f_from_coro_f+ async def test_download_chunked_content(self):request = Request(self.getURL("chunked"))- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"chunked content\n")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b"chunked content\n"- def test_download_broken_content_cause_data_loss(self, url="broken"):+ @deferred_f_from_coro_f+ async def test_download_broken_content_cause_data_loss(+ self, url: str = "broken"+ ) -> None:# TODO: this one checks for Twisted-specific exceptionsrequest = Request(self.getURL(url))- d = self.download_request(request, Spider("foo"))-- def checkDataLoss(failure):- if failure.check(ResponseFailed) and any(- r.check(_DataLoss) for r in failure.value.reasons- ):- return None- return failure-- d.addCallback(lambda _: self.fail("No DataLoss exception"))- d.addErrback(checkDataLoss)- return d+ with pytest.raises(ResponseFailed) as exc_info:+ await self.download_request(request, Spider("foo"))+ assert any(r.check(_DataLoss) for r in exc_info.value.reasons)def test_download_broken_chunked_content_cause_data_loss(self):return self.test_download_broken_content_cause_data_loss("broken-chunked")- def test_download_broken_content_allow_data_loss(self, url="broken"):+ @deferred_f_from_coro_f+ async def test_download_broken_content_allow_data_loss(+ self, url: str = "broken"+ ) -> None:request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False})- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.flags)- d.addCallback(self.assertEqual, ["dataloss"])- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.flags == ["dataloss"]def test_download_broken_chunked_content_allow_data_loss(self):return self.test_download_broken_content_allow_data_loss("broken-chunked")- def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"):+ @deferred_f_from_coro_f+ async def test_download_broken_content_allow_data_loss_via_setting(+ self, url: str = "broken"+ ) -> None:crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False})download_handler = build_from_crawler(self.download_handler_cls, crawler)request = Request(self.getURL(url))- d = download_handler.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.flags)- d.addCallback(self.assertEqual, ["dataloss"])- return d+ response = await maybe_deferred_to_future(+ download_handler.download_request(request, Spider("foo"))+ )+ assert response.flags == ["dataloss"]def test_download_broken_chunked_content_allow_data_loss_via_setting(self):return self.test_download_broken_content_allow_data_loss_via_setting("broken-chunked")- def test_protocol(self):+ @deferred_f_from_coro_f+ async def test_protocol(self):request = Request(self.getURL("host"), method="GET")- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.protocol)- d.addCallback(self.assertEqual, "HTTP/1.1")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.protocol == "HTTP/1.1"class TestHttps11Base(TestHttp11Base):@@ -481,8 +475,8 @@ class TestHttps11Base(TestHttp11Base):'subject "/C=IE/O=Scrapy/CN=localhost"')- @defer.inlineCallbacks- def test_tls_logging(self):+ @deferred_f_from_coro_f+ async def test_tls_logging(self):crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True})@@ -490,15 +484,15 @@ class TestHttps11Base(TestHttp11Base):try:with LogCapture() as log_capture:request = Request(self.getURL("file"))- d = download_handler.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"0123456789")- yield d+ response = await maybe_deferred_to_future(+ download_handler.download_request(request, Spider("foo"))+ )+ assert response.body == b"0123456789"log_capture.check_present(("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message))finally:- yield download_handler.close()+ await maybe_deferred_to_future(maybeDeferred(download_handler.close))class TestSimpleHttpsBase(unittest.TestCase, ABC):@@ -536,7 +530,6 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):settings_dict = Nonecrawler = get_crawler(settings_dict=settings_dict)self.download_handler = build_from_crawler(self.download_handler_cls, crawler)- self.download_request = self.download_handler.download_request@defer.inlineCallbacksdef tearDown(self):@@ -548,12 +541,16 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):def getURL(self, path):return f"https://{self.host}:{self.portno}/{path}"- def test_download(self):+ async def download_request(self, request: Request, spider: Spider) -> Response:+ return await maybe_deferred_to_future(+ self.download_handler.download_request(request, spider)+ )++ @deferred_f_from_coro_f+ async def test_download(self):request = Request(self.getURL("file"))- d = self.download_request(request, Spider("foo"))- d.addCallback(lambda r: r.body)- d.addCallback(self.assertEqual, b"0123456789")- return d+ response = await self.download_request(request, Spider("foo"))+ assert response.body == b"0123456789"class TestHttpsWrongHostnameBase(TestSimpleHttpsBase):@@ -604,25 +601,29 @@ class TestHttpMockServerBase(unittest.TestCase, ABC):def tearDownClass(cls):cls.mockserver.__exit__(None, None, None)- @defer.inlineCallbacks- def test_download_with_content_length(self):+ @deferred_f_from_coro_f+ async def test_download_with_content_length(self):crawler = get_crawler(SingleRequestSpider, self.settings_dict)# http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid# download it- yield crawler.crawl(- seed=Request(- url=self.mockserver.url("/partial", is_secure=self.is_secure),- meta={"download_maxsize": 1000},+ await maybe_deferred_to_future(+ crawler.crawl(+ seed=Request(+ url=self.mockserver.url("/partial", is_secure=self.is_secure),+ meta={"download_maxsize": 1000},+ )))failure = crawler.spider.meta["failure"]assert isinstance(failure.value, defer.CancelledError)- @defer.inlineCallbacks- def test_download(self):+ @deferred_f_from_coro_f+ async def test_download(self):crawler = get_crawler(SingleRequestSpider, self.settings_dict)- yield crawler.crawl(- seed=Request(url=self.mockserver.url("", is_secure=self.is_secure))+ await maybe_deferred_to_future(+ crawler.crawl(+ seed=Request(url=self.mockserver.url("", is_secure=self.is_secure))+ ))failure = crawler.spider.meta.get("failure")assert failure is None@@ -663,7 +664,6 @@ class TestHttpProxyBase(unittest.TestCase, ABC):self.download_handler = build_from_crawler(self.download_handler_cls, get_crawler())- self.download_request = self.download_handler.download_request@defer.inlineCallbacksdef tearDown(self):@@ -674,42 +674,44 @@ class TestHttpProxyBase(unittest.TestCase, ABC):def getURL(self, path):return f"http://127.0.0.1:{self.portno}/{path}"- def test_download_with_proxy(self):- def _test(response):- assert response.status == 200- assert response.url == request.url- assert response.body == self.expected_http_proxy_request_body+ async def download_request(self, request: Request, spider: Spider) -> Response:+ return await maybe_deferred_to_future(+ self.download_handler.download_request(request, spider)+ )+ @deferred_f_from_coro_f+ async def test_download_with_proxy(self):http_proxy = self.getURL("")request = Request("http://example.com", meta={"proxy": http_proxy})- return self.download_request(request, Spider("foo")).addCallback(_test)-- def test_download_without_proxy(self):- def _test(response):- assert response.status == 200- assert response.url == request.url- assert response.body == b"/path/to/resource"+ response = await self.download_request(request, Spider("foo"))+ assert response.status == 200+ assert response.url == request.url+ assert response.body == self.expected_http_proxy_request_body+ @deferred_f_from_coro_f+ async def test_download_without_proxy(self):request = Request(self.getURL("path/to/resource"))- return self.download_request(request, Spider("foo")).addCallback(_test)+ response = await self.download_request(request, Spider("foo"))+ assert response.status == 200+ assert response.url == request.url+ assert response.body == b"/path/to/resource"- @defer.inlineCallbacks- def test_download_with_proxy_https_timeout(self):+ @deferred_f_from_coro_f+ async def test_download_with_proxy_https_timeout(self):if NON_EXISTING_RESOLVABLE:pytest.skip("Non-existing hosts are resolvable")http_proxy = self.getURL("")domain = "https://no-such-domain.nosuch"request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})- d = self.download_request(request, Spider("foo"))- timeout = yield self.assertFailure(d, error.TimeoutError)- assert domain in timeout.osError-- def test_download_with_proxy_without_http_scheme(self):- def _test(response):- assert response.status == 200- assert response.url == request.url- assert response.body == self.expected_http_proxy_request_body+ with pytest.raises(error.TimeoutError) as exc_info:+ await self.download_request(request, Spider("foo"))+ assert domain in exc_info.value.osError+ @deferred_f_from_coro_f+ async def test_download_with_proxy_without_http_scheme(self):http_proxy = self.getURL("").replace("http://", "")request = Request("http://example.com", meta={"proxy": http_proxy})- return self.download_request(request, Spider("foo")).addCallback(_test)+ response = await self.download_request(request, Spider("foo"))+ assert response.status == 200+ assert response.url == request.url+ assert response.body == self.expected_http_proxy_request_body