Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rewrite download handler tests to coroutines. (#6846)

Andrey Rakhmatullin8ae418df44b7
diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py
index 807c8c4cb..bc306aa07 100644
--- a/tests/test_downloader_handler_twisted_http10.py
+++ b/tests/test_downloader_handler_twisted_http10.py
@@ -9,6 +9,7 @@ import pytest
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.http import Request
from scrapy.spiders import Spider
+from scrapy.utils.defer import deferred_f_from_coro_f
from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase
if TYPE_CHECKING:
@@ -25,12 +26,11 @@ class HTTP10DownloadHandlerMixin:
class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase):
"""HTTP 1.0 test case"""
- def test_protocol(self):
+ @deferred_f_from_coro_f
+ async def test_protocol(self):
request = Request(self.getURL("host"), method="GET")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.protocol)
- d.addCallback(self.assertEqual, "HTTP/1.0")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.protocol == "HTTP/1.0"
class TestHttps10(TestHttp10):
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 159f403d0..e058cedae 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -15,6 +15,10 @@ from twisted.web.http import H2_ENABLED
from scrapy.http import Request
from scrapy.spiders import Spider
+from scrapy.utils.defer import (
+ deferred_f_from_coro_f,
+ maybe_deferred_to_future,
+)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver import ssl_context_factory
@@ -50,15 +54,14 @@ class H2DownloadHandlerMixin:
class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
- def test_protocol(self):
+ @deferred_f_from_coro_f
+ async def test_protocol(self):
request = Request(self.getURL("host"), method="GET")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.protocol)
- d.addCallback(self.assertEqual, "h2")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.protocol == "h2"
- @defer.inlineCallbacks
- def test_download_with_maxsize_very_large_file(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_maxsize_very_large_file(self):
from twisted.internet import reactor
with mock.patch("scrapy.core.http2.stream.logger") as logger:
@@ -67,8 +70,10 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
def check(logger):
logger.error.assert_called_once_with(mock.ANY)
- d = self.download_request(request, Spider("foo", download_maxsize=1500))
- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+ await self.download_request(
+ request, Spider("foo", download_maxsize=1500)
+ )
# As the error message is logged in the dataReceived callback, we
# have to give a bit of time to the reactor to process the queue
@@ -76,13 +81,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
d = defer.Deferred()
d.addCallback(check)
reactor.callLater(0.1, d.callback, logger)
- yield d
+ await maybe_deferred_to_future(d)
- @defer.inlineCallbacks
- def test_unsupported_scheme(self):
+ @deferred_f_from_coro_f
+ async def test_unsupported_scheme(self):
request = Request("ftp://unsupported.scheme")
- d = self.download_request(request, Spider("foo"))
- yield self.assertFailure(d, SchemeNotSupported)
+ with pytest.raises(SchemeNotSupported):
+ await self.download_request(request, Spider("foo"))
def test_download_broken_content_cause_data_loss(self, url="broken"):
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
@@ -102,70 +107,60 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
- def test_concurrent_requests_same_domain(self):
+ @deferred_f_from_coro_f
+ async def test_concurrent_requests_same_domain(self):
spider = Spider("foo")
request1 = Request(self.getURL("file"))
- d1 = self.download_request(request1, spider)
- d1.addCallback(lambda r: r.body)
- d1.addCallback(self.assertEqual, b"0123456789")
+ response1 = await self.download_request(request1, spider)
+ assert response1.body == b"0123456789"
request2 = Request(self.getURL("echo"), method="POST")
- d2 = self.download_request(request2, spider)
- d2.addCallback(lambda r: r.headers["Content-Length"])
- d2.addCallback(self.assertEqual, b"79")
-
- return defer.DeferredList([d1, d2])
+ response2 = await self.download_request(request2, spider)
+ assert response2.headers["Content-Length"] == b"79"
@pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")
- def test_connect_request(self):
+ @deferred_f_from_coro_f
+ async def test_connect_request(self):
request = Request(self.getURL("file"), method="CONNECT")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b""
- def test_custom_content_length_good(self):
+ @deferred_f_from_coro_f
+ async def test_custom_content_length_good(self):
request = Request(self.getURL("contentlength"))
custom_content_length = str(len(request.body))
request.headers["Content-Length"] = custom_content_length
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.text)
- d.addCallback(self.assertEqual, custom_content_length)
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.text == custom_content_length
- def test_custom_content_length_bad(self):
+ @deferred_f_from_coro_f
+ async def test_custom_content_length_bad(self):
request = Request(self.getURL("contentlength"))
actual_content_length = str(len(request.body))
bad_content_length = str(len(request.body) + 1)
request.headers["Content-Length"] = bad_content_length
- log = LogCapture()
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.text)
- d.addCallback(self.assertEqual, actual_content_length)
- d.addCallback(
- lambda _: log.check_present(
- (
- "scrapy.core.http2.stream",
- "WARNING",
- f"Ignoring bad Content-Length header "
- f"{bad_content_length!r} of request {request}, sending "
- f"{actual_content_length!r} instead",
- )
+ with LogCapture() as log:
+ response = await self.download_request(request, Spider("foo"))
+ assert response.text == actual_content_length
+ log.check_present(
+ (
+ "scrapy.core.http2.stream",
+ "WARNING",
+ f"Ignoring bad Content-Length header "
+ f"{bad_content_length!r} of request {request}, sending "
+ f"{actual_content_length!r} instead",
)
)
- d.addCallback(lambda _: log.uninstall())
- return d
- def test_duplicate_header(self):
+ @deferred_f_from_coro_f
+ async def test_duplicate_header(self):
request = Request(self.getURL("echo"))
header, value1, value2 = "Custom-Header", "foo", "bar"
request.headers.appendlist(header, value1)
request.headers.appendlist(header, value2)
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: json.loads(r.text)["headers"][header])
- d.addCallback(self.assertEqual, [value1, value2])
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert json.loads(response.text)["headers"][header] == [value1, value2]
class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
@@ -222,12 +217,13 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase):
self.download_handler = build_from_crawler(
self.download_handler_cls, get_crawler()
)
- self.download_request = self.download_handler.download_request
def getURL(self, path):
return f"{self.scheme}://{self.host}:{self.portno}/{path}"
- @defer.inlineCallbacks
- def test_download_with_proxy_https_timeout(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_proxy_https_timeout(self):
with pytest.raises(NotImplementedError):
- yield super().test_download_with_proxy_https_timeout()
+ await maybe_deferred_to_future(
+ super().test_download_with_proxy_https_timeout()
+ )
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index dacadb075..09cdbaf35 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -22,10 +22,14 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured
-from scrapy.http import HtmlResponse, Request
+from scrapy.http import HtmlResponse, Request, Response
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
+from scrapy.utils.defer import (
+ deferred_f_from_coro_f,
+ maybe_deferred_to_future,
+)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
@@ -95,28 +99,33 @@ class TestFile(unittest.TestCase):
# add a special char to check that they are handled correctly
self.fd, self.tmpname = mkstemp(suffix="^")
Path(self.tmpname).write_text("0123456789", encoding="utf-8")
- handler = build_from_crawler(FileDownloadHandler, get_crawler())
- self.download_request = handler.download_request
+ self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler())
def tearDown(self):
os.close(self.fd)
Path(self.tmpname).unlink()
- def test_download(self):
- def _test(response):
- assert response.url == request.url
- assert response.status == 200
- assert response.body == b"0123456789"
- assert response.protocol is None
+ async def download_request(self, request: Request, spider: Spider) -> Response:
+ return await maybe_deferred_to_future(
+ self.download_handler.download_request(request, spider)
+ )
+ @deferred_f_from_coro_f
+ async def test_download(self):
request = Request(path_to_file_uri(self.tmpname))
assert request.url.upper().endswith("%5E")
- return self.download_request(request, Spider("foo")).addCallback(_test)
-
- def test_non_existent(self):
+ response = await self.download_request(request, Spider("foo"))
+ assert response.url == request.url
+ assert response.status == 200
+ assert response.body == b"0123456789"
+ assert response.protocol is None
+
+ @deferred_f_from_coro_f
+ async def test_non_existent(self):
request = Request(path_to_file_uri(mkdtemp()))
- d = self.download_request(request, Spider("foo"))
- return self.assertFailure(d, OSError)
+ # the specific exception differs between platforms
+ with pytest.raises(OSError): # noqa: PT011
+ await self.download_request(request, Spider("foo"))
class HttpDownloadHandlerMock:
@@ -479,69 +488,65 @@ class TestDataURI(unittest.TestCase):
def setUp(self):
crawler = get_crawler()
self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler)
- self.download_request = self.download_handler.download_request
self.spider = Spider("foo")
- def test_response_attrs(self):
- uri = "data:,A%20brief%20note"
-
- def _test(response):
- assert response.url == uri
- assert not response.headers
+ async def download_request(self, request: Request, spider: Spider) -> Response:
+ return await maybe_deferred_to_future(
+ self.download_handler.download_request(request, spider)
+ )
+ @deferred_f_from_coro_f
+ async def test_response_attrs(self):
+ uri = "data:,A%20brief%20note"
request = Request(uri)
- return self.download_request(request, self.spider).addCallback(_test)
-
- def test_default_mediatype_encoding(self):
- def _test(response):
- assert response.text == "A brief note"
- assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
- assert response.encoding == "US-ASCII"
+ response = await self.download_request(request, self.spider)
+ assert response.url == uri
+ assert not response.headers
+ @deferred_f_from_coro_f
+ async def test_default_mediatype_encoding(self):
request = Request("data:,A%20brief%20note")
- return self.download_request(request, self.spider).addCallback(_test)
-
- def test_default_mediatype(self):
- def _test(response):
- assert response.text == "\u038e\u03a3\u038e"
- assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
- assert response.encoding == "iso-8859-7"
+ response = await self.download_request(request, self.spider)
+ assert response.text == "A brief note"
+ assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
+ assert response.encoding == "US-ASCII"
+ @deferred_f_from_coro_f
+ async def test_default_mediatype(self):
request = Request("data:;charset=iso-8859-7,%be%d3%be")
- return self.download_request(request, self.spider).addCallback(_test)
-
- def test_text_charset(self):
- def _test(response):
- assert response.text == "\u038e\u03a3\u038e"
- assert response.body == b"\xbe\xd3\xbe"
- assert response.encoding == "iso-8859-7"
+ response = await self.download_request(request, self.spider)
+ assert response.text == "\u038e\u03a3\u038e"
+ assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
+ assert response.encoding == "iso-8859-7"
+ @deferred_f_from_coro_f
+ async def test_text_charset(self):
request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
- return self.download_request(request, self.spider).addCallback(_test)
-
- def test_mediatype_parameters(self):
- def _test(response):
- assert response.text == "\u038e\u03a3\u038e"
- assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
- assert response.encoding == "utf-8"
+ response = await self.download_request(request, self.spider)
+ assert response.text == "\u038e\u03a3\u038e"
+ assert response.body == b"\xbe\xd3\xbe"
+ assert response.encoding == "iso-8859-7"
+ @deferred_f_from_coro_f
+ async def test_mediatype_parameters(self):
request = Request(
"data:text/plain;foo=%22foo;bar%5C%22%22;"
"charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22"
",%CE%8E%CE%A3%CE%8E"
)
- return self.download_request(request, self.spider).addCallback(_test)
-
- def test_base64(self):
- def _test(response):
- assert response.text == "Hello, world."
+ response = await self.download_request(request, self.spider)
+ assert response.text == "\u038e\u03a3\u038e"
+ assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
+ assert response.encoding == "utf-8"
+ @deferred_f_from_coro_f
+ async def test_base64(self):
request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D")
- return self.download_request(request, self.spider).addCallback(_test)
-
- def test_protocol(self):
- def _test(response):
- assert response.protocol is None
+ response = await self.download_request(request, self.spider)
+ assert response.text == "Hello, world."
+ @deferred_f_from_coro_f
+ async def test_protocol(self):
request = Request("data:,")
- return self.download_request(request, self.spider).addCallback(_test)
+ response = await self.download_request(request, self.spider)
+ assert response.protocol is None
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 5eaf66966..14e12a3e6 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
import shutil
import sys
from abc import ABC, abstractmethod
@@ -13,14 +14,20 @@ from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer, error
+from twisted.internet.defer import maybeDeferred
from twisted.protocols.policies import WrappingFactory
from twisted.trial import unittest
from twisted.web import resource, server, static, util
from twisted.web._newclient import ResponseFailed
from twisted.web.http import _DataLoss
-from scrapy.http import Headers, HtmlResponse, Request, TextResponse
+from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
from scrapy.spiders import Spider
+from scrapy.utils.defer import (
+ deferred_f_from_coro_f,
+ deferred_from_coro,
+ maybe_deferred_to_future,
+)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
@@ -178,7 +185,6 @@ class TestHttpBase(unittest.TestCase, ABC):
self.download_handler = build_from_crawler(
self.download_handler_cls, get_crawler()
)
- self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
def tearDown(self):
@@ -190,36 +196,37 @@ class TestHttpBase(unittest.TestCase, ABC):
def getURL(self, path):
return f"{self.scheme}://{self.host}:{self.portno}/{path}"
- def test_download(self):
+ async def download_request(self, request: Request, spider: Spider) -> Response:
+ return await maybe_deferred_to_future(
+ self.download_handler.download_request(request, spider)
+ )
+
+ @deferred_f_from_coro_f
+ async def test_download(self):
request = Request(self.getURL("file"))
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"0123456789")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b"0123456789"
- def test_download_head(self):
+ @deferred_f_from_coro_f
+ async def test_download_head(self):
request = Request(self.getURL("file"), method="HEAD")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b""
- def test_redirect_status(self):
+ @deferred_f_from_coro_f
+ async def test_redirect_status(self):
request = Request(self.getURL("redirect"))
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.status)
- d.addCallback(self.assertEqual, 302)
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.status == 302
- def test_redirect_status_head(self):
+ @deferred_f_from_coro_f
+ async def test_redirect_status_head(self):
request = Request(self.getURL("redirect"), method="HEAD")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.status)
- d.addCallback(self.assertEqual, 302)
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.status == 302
- @defer.inlineCallbacks
- def test_timeout_download_from_spider_nodata_rcvd(self):
+ @deferred_f_from_coro_f
+ async def test_timeout_download_from_spider_nodata_rcvd(self):
if self.reactor_pytest != "default" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
raise unittest.SkipTest(
@@ -230,11 +237,12 @@ class TestHttpBase(unittest.TestCase, ABC):
spider = Spider("foo")
meta = {"download_timeout": 0.5}
request = Request(self.getURL("wait"), meta=meta)
- d = self.download_request(request, spider)
- yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
+ d = deferred_from_coro(self.download_request(request, spider))
+ with pytest.raises((defer.TimeoutError, error.TimeoutError)):
+ await maybe_deferred_to_future(d)
- @defer.inlineCallbacks
- def test_timeout_download_from_spider_server_hangs(self):
+ @deferred_f_from_coro_f
+ async def test_timeout_download_from_spider_server_hangs(self):
if self.reactor_pytest != "default" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
raise unittest.SkipTest(
@@ -244,28 +252,27 @@ class TestHttpBase(unittest.TestCase, ABC):
spider = Spider("foo")
meta = {"download_timeout": 0.5}
request = Request(self.getURL("hang-after-headers"), meta=meta)
- d = self.download_request(request, spider)
- yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
-
- def test_host_header_not_in_request_headers(self):
- def _test(response):
- assert response.body == to_bytes(f"{self.host}:{self.portno}")
- assert not request.headers
+ d = deferred_from_coro(self.download_request(request, spider))
+ with pytest.raises((defer.TimeoutError, error.TimeoutError)):
+ await maybe_deferred_to_future(d)
+ @deferred_f_from_coro_f
+ async def test_host_header_not_in_request_headers(self):
request = Request(self.getURL("host"))
- return self.download_request(request, Spider("foo")).addCallback(_test)
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == to_bytes(f"{self.host}:{self.portno}")
+ assert not request.headers
- def test_host_header_seted_in_request_headers(self):
+ @deferred_f_from_coro_f
+ async def test_host_header_set_in_request_headers(self):
host = self.host + ":" + str(self.portno)
-
- def _test(response):
- assert response.body == host.encode()
- assert request.headers.get("Host") == host.encode()
-
request = Request(self.getURL("host"), headers={"Host": host})
- return self.download_request(request, Spider("foo")).addCallback(_test)
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == host.encode()
+ assert request.headers.get("Host") == host.encode()
- def test_content_length_zero_bodyless_post_request_headers(self):
+ @deferred_f_from_coro_f
+ async def test_content_length_zero_bodyless_post_request_headers(self):
"""Tests if "Content-Length: 0" is sent for bodyless POST requests.
This is not strictly required by HTTP RFCs but can cause trouble
@@ -276,105 +283,93 @@ class TestHttpBase(unittest.TestCase, ABC):
https://github.com/kennethreitz/requests/issues/405
https://bugs.python.org/issue14721
"""
-
- def _test(response):
- assert response.body == b"0"
-
request = Request(self.getURL("contentlength"), method="POST")
- return self.download_request(request, Spider("foo")).addCallback(_test)
-
- def test_content_length_zero_bodyless_post_only_one(self):
- def _test(response):
- import json
-
- headers = Headers(json.loads(response.text)["headers"])
- contentlengths = headers.getlist("Content-Length")
- assert len(contentlengths) == 1
- assert contentlengths == [b"0"]
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b"0"
+ @deferred_f_from_coro_f
+ async def test_content_length_zero_bodyless_post_only_one(self):
request = Request(self.getURL("echo"), method="POST")
- return self.download_request(request, Spider("foo")).addCallback(_test)
-
- def test_payload(self):
+ response = await self.download_request(request, Spider("foo"))
+ headers = Headers(json.loads(response.text)["headers"])
+ contentlengths = headers.getlist("Content-Length")
+ assert len(contentlengths) == 1
+ assert contentlengths == [b"0"]
+
+ @deferred_f_from_coro_f
+ async def test_payload(self):
body = b"1" * 100 # PayloadResource requires body length to be 100
request = Request(self.getURL("payload"), method="POST", body=body)
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, body)
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == body
- def test_response_header_content_length(self):
+ @deferred_f_from_coro_f
+ async def test_response_header_content_length(self):
request = Request(self.getURL("file"), method=b"GET")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.headers[b"content-length"])
- d.addCallback(self.assertEqual, b"159")
- return d
-
- def _test_response_class(self, filename, body, response_class):
- def _test(response):
- assert type(response) is response_class # pylint: disable=unidiomatic-typecheck
+ response = await self.download_request(request, Spider("foo"))
+ assert response.headers[b"content-length"] == b"159"
+ async def _test_response_class(
+ self, filename: str, body: bytes, response_class: type[Response]
+ ) -> None:
request = Request(self.getURL(filename), body=body)
- return self.download_request(request, Spider("foo")).addCallback(_test)
+ response = await self.download_request(request, Spider("foo"))
+ assert type(response) is response_class # pylint: disable=unidiomatic-typecheck
- def test_response_class_from_url(self):
- return self._test_response_class("foo.html", b"", HtmlResponse)
+ @deferred_f_from_coro_f
+ async def test_response_class_from_url(self):
+ await self._test_response_class("foo.html", b"", HtmlResponse)
- def test_response_class_from_body(self):
- return self._test_response_class(
+ @deferred_f_from_coro_f
+ async def test_response_class_from_body(self):
+ await self._test_response_class(
"foo",
b"<!DOCTYPE html>\n<title>.</title>",
HtmlResponse,
)
- def test_get_duplicate_header(self):
- def _test(response):
- assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]
-
+ @deferred_f_from_coro_f
+ async def test_get_duplicate_header(self):
request = Request(self.getURL("duplicate-header"))
- return self.download_request(request, Spider("foo")).addCallback(_test)
+ response = await self.download_request(request, Spider("foo"))
+ assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"]
class TestHttp11Base(TestHttpBase):
"""HTTP 1.1 test case"""
- def test_download_without_maxsize_limit(self):
+ @deferred_f_from_coro_f
+ async def test_download_without_maxsize_limit(self):
request = Request(self.getURL("file"))
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"0123456789")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b"0123456789"
- def test_response_class_choosing_request(self):
+ @deferred_f_from_coro_f
+ async def test_response_class_choosing_request(self):
"""Tests choosing of correct response type
in case of Content-Type is empty but body contains text.
"""
body = b"Some plain text\ndata with tabs\t and null bytes\0"
-
- def _test_type(response):
- assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck
-
request = Request(self.getURL("nocontenttype"), body=body)
- d = self.download_request(request, Spider("foo"))
- d.addCallback(_test_type)
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck
- @defer.inlineCallbacks
- def test_download_with_maxsize(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_maxsize(self):
request = Request(self.getURL("file"))
# 10 is minimal size for this request and the limit is only counted on
# response body. (regardless of headers)
- d = self.download_request(request, Spider("foo", download_maxsize=10))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"0123456789")
- yield d
+ response = await self.download_request(
+ request, Spider("foo", download_maxsize=10)
+ )
+ assert response.body == b"0123456789"
- d = self.download_request(request, Spider("foo", download_maxsize=9))
- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+ await self.download_request(request, Spider("foo", download_maxsize=9))
- @defer.inlineCallbacks
- def test_download_with_maxsize_very_large_file(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_maxsize_very_large_file(self):
from twisted.internet import reactor
# TODO: the logger check is specific to scrapy.core.downloader.handlers.http11
@@ -384,8 +379,10 @@ class TestHttp11Base(TestHttpBase):
def check(logger):
logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
- d = self.download_request(request, Spider("foo", download_maxsize=1500))
- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+ await self.download_request(
+ request, Spider("foo", download_maxsize=1500)
+ )
# As the error message is logged in the dataReceived callback, we
# have to give a bit of time to the reactor to process the queue
@@ -393,84 +390,81 @@ class TestHttp11Base(TestHttpBase):
d = defer.Deferred()
d.addCallback(check)
reactor.callLater(0.1, d.callback, logger)
- yield d
+ await maybe_deferred_to_future(d)
- @defer.inlineCallbacks
- def test_download_with_maxsize_per_req(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_maxsize_per_req(self):
meta = {"download_maxsize": 2}
request = Request(self.getURL("file"), meta=meta)
- d = self.download_request(request, Spider("foo"))
- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+ await self.download_request(request, Spider("foo"))
- @defer.inlineCallbacks
- def test_download_with_small_maxsize_per_spider(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_small_maxsize_per_spider(self):
request = Request(self.getURL("file"))
- d = self.download_request(request, Spider("foo", download_maxsize=2))
- yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+ with pytest.raises((defer.CancelledError, error.ConnectionAborted)):
+ await self.download_request(request, Spider("foo", download_maxsize=2))
- def test_download_with_large_maxsize_per_spider(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_large_maxsize_per_spider(self):
request = Request(self.getURL("file"))
- d = self.download_request(request, Spider("foo", download_maxsize=100))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"0123456789")
- return d
+ response = await self.download_request(
+ request, Spider("foo", download_maxsize=100)
+ )
+ assert response.body == b"0123456789"
- def test_download_chunked_content(self):
+ @deferred_f_from_coro_f
+ async def test_download_chunked_content(self):
request = Request(self.getURL("chunked"))
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"chunked content\n")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b"chunked content\n"
- def test_download_broken_content_cause_data_loss(self, url="broken"):
+ @deferred_f_from_coro_f
+ async def test_download_broken_content_cause_data_loss(
+ self, url: str = "broken"
+ ) -> None:
# TODO: this one checks for Twisted-specific exceptions
request = Request(self.getURL(url))
- d = self.download_request(request, Spider("foo"))
-
- def checkDataLoss(failure):
- if failure.check(ResponseFailed) and any(
- r.check(_DataLoss) for r in failure.value.reasons
- ):
- return None
- return failure
-
- d.addCallback(lambda _: self.fail("No DataLoss exception"))
- d.addErrback(checkDataLoss)
- return d
+ with pytest.raises(ResponseFailed) as exc_info:
+ await self.download_request(request, Spider("foo"))
+ assert any(r.check(_DataLoss) for r in exc_info.value.reasons)
def test_download_broken_chunked_content_cause_data_loss(self):
return self.test_download_broken_content_cause_data_loss("broken-chunked")
- def test_download_broken_content_allow_data_loss(self, url="broken"):
+ @deferred_f_from_coro_f
+ async def test_download_broken_content_allow_data_loss(
+ self, url: str = "broken"
+ ) -> None:
request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False})
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.flags)
- d.addCallback(self.assertEqual, ["dataloss"])
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.flags == ["dataloss"]
def test_download_broken_chunked_content_allow_data_loss(self):
return self.test_download_broken_content_allow_data_loss("broken-chunked")
- def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"):
+ @deferred_f_from_coro_f
+ async def test_download_broken_content_allow_data_loss_via_setting(
+ self, url: str = "broken"
+ ) -> None:
crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False})
download_handler = build_from_crawler(self.download_handler_cls, crawler)
request = Request(self.getURL(url))
- d = download_handler.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.flags)
- d.addCallback(self.assertEqual, ["dataloss"])
- return d
+ response = await maybe_deferred_to_future(
+ download_handler.download_request(request, Spider("foo"))
+ )
+ assert response.flags == ["dataloss"]
def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
return self.test_download_broken_content_allow_data_loss_via_setting(
"broken-chunked"
)
- def test_protocol(self):
+ @deferred_f_from_coro_f
+ async def test_protocol(self):
request = Request(self.getURL("host"), method="GET")
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.protocol)
- d.addCallback(self.assertEqual, "HTTP/1.1")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.protocol == "HTTP/1.1"
class TestHttps11Base(TestHttp11Base):
@@ -481,8 +475,8 @@ class TestHttps11Base(TestHttp11Base):
'subject "/C=IE/O=Scrapy/CN=localhost"'
)
- @defer.inlineCallbacks
- def test_tls_logging(self):
+ @deferred_f_from_coro_f
+ async def test_tls_logging(self):
crawler = get_crawler(
settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}
)
@@ -490,15 +484,15 @@ class TestHttps11Base(TestHttp11Base):
try:
with LogCapture() as log_capture:
request = Request(self.getURL("file"))
- d = download_handler.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"0123456789")
- yield d
+ response = await maybe_deferred_to_future(
+ download_handler.download_request(request, Spider("foo"))
+ )
+ assert response.body == b"0123456789"
log_capture.check_present(
("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message)
)
finally:
- yield download_handler.close()
+ await maybe_deferred_to_future(maybeDeferred(download_handler.close))
class TestSimpleHttpsBase(unittest.TestCase, ABC):
@@ -536,7 +530,6 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):
settings_dict = None
crawler = get_crawler(settings_dict=settings_dict)
self.download_handler = build_from_crawler(self.download_handler_cls, crawler)
- self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
def tearDown(self):
@@ -548,12 +541,16 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):
def getURL(self, path):
return f"https://{self.host}:{self.portno}/{path}"
- def test_download(self):
+ async def download_request(self, request: Request, spider: Spider) -> Response:
+ return await maybe_deferred_to_future(
+ self.download_handler.download_request(request, spider)
+ )
+
+ @deferred_f_from_coro_f
+ async def test_download(self):
request = Request(self.getURL("file"))
- d = self.download_request(request, Spider("foo"))
- d.addCallback(lambda r: r.body)
- d.addCallback(self.assertEqual, b"0123456789")
- return d
+ response = await self.download_request(request, Spider("foo"))
+ assert response.body == b"0123456789"
class TestHttpsWrongHostnameBase(TestSimpleHttpsBase):
@@ -604,25 +601,29 @@ class TestHttpMockServerBase(unittest.TestCase, ABC):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
- @defer.inlineCallbacks
- def test_download_with_content_length(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_content_length(self):
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
# http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
# download it
- yield crawler.crawl(
- seed=Request(
- url=self.mockserver.url("/partial", is_secure=self.is_secure),
- meta={"download_maxsize": 1000},
+ await maybe_deferred_to_future(
+ crawler.crawl(
+ seed=Request(
+ url=self.mockserver.url("/partial", is_secure=self.is_secure),
+ meta={"download_maxsize": 1000},
+ )
)
)
failure = crawler.spider.meta["failure"]
assert isinstance(failure.value, defer.CancelledError)
- @defer.inlineCallbacks
- def test_download(self):
+ @deferred_f_from_coro_f
+ async def test_download(self):
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
- yield crawler.crawl(
- seed=Request(url=self.mockserver.url("", is_secure=self.is_secure))
+ await maybe_deferred_to_future(
+ crawler.crawl(
+ seed=Request(url=self.mockserver.url("", is_secure=self.is_secure))
+ )
)
failure = crawler.spider.meta.get("failure")
assert failure is None
@@ -663,7 +664,6 @@ class TestHttpProxyBase(unittest.TestCase, ABC):
self.download_handler = build_from_crawler(
self.download_handler_cls, get_crawler()
)
- self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
def tearDown(self):
@@ -674,42 +674,44 @@ class TestHttpProxyBase(unittest.TestCase, ABC):
def getURL(self, path):
return f"http://127.0.0.1:{self.portno}/{path}"
- def test_download_with_proxy(self):
- def _test(response):
- assert response.status == 200
- assert response.url == request.url
- assert response.body == self.expected_http_proxy_request_body
+ async def download_request(self, request: Request, spider: Spider) -> Response:
+ return await maybe_deferred_to_future(
+ self.download_handler.download_request(request, spider)
+ )
+ @deferred_f_from_coro_f
+ async def test_download_with_proxy(self):
http_proxy = self.getURL("")
request = Request("http://example.com", meta={"proxy": http_proxy})
- return self.download_request(request, Spider("foo")).addCallback(_test)
-
- def test_download_without_proxy(self):
- def _test(response):
- assert response.status == 200
- assert response.url == request.url
- assert response.body == b"/path/to/resource"
+ response = await self.download_request(request, Spider("foo"))
+ assert response.status == 200
+ assert response.url == request.url
+ assert response.body == self.expected_http_proxy_request_body
+ @deferred_f_from_coro_f
+ async def test_download_without_proxy(self):
request = Request(self.getURL("path/to/resource"))
- return self.download_request(request, Spider("foo")).addCallback(_test)
+ response = await self.download_request(request, Spider("foo"))
+ assert response.status == 200
+ assert response.url == request.url
+ assert response.body == b"/path/to/resource"
- @defer.inlineCallbacks
- def test_download_with_proxy_https_timeout(self):
+ @deferred_f_from_coro_f
+ async def test_download_with_proxy_https_timeout(self):
if NON_EXISTING_RESOLVABLE:
pytest.skip("Non-existing hosts are resolvable")
http_proxy = self.getURL("")
domain = "https://no-such-domain.nosuch"
request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})
- d = self.download_request(request, Spider("foo"))
- timeout = yield self.assertFailure(d, error.TimeoutError)
- assert domain in timeout.osError
-
- def test_download_with_proxy_without_http_scheme(self):
- def _test(response):
- assert response.status == 200
- assert response.url == request.url
- assert response.body == self.expected_http_proxy_request_body
+ with pytest.raises(error.TimeoutError) as exc_info:
+ await self.download_request(request, Spider("foo"))
+ assert domain in exc_info.value.osError
+ @deferred_f_from_coro_f
+ async def test_download_with_proxy_without_http_scheme(self):
http_proxy = self.getURL("").replace("http://", "")
request = Request("http://example.com", meta={"proxy": http_proxy})
- return self.download_request(request, Spider("foo")).addCallback(_test)
+ response = await self.download_request(request, Spider("foo"))
+ assert response.status == 200
+ assert response.url == request.url
+ assert response.body == self.expected_http_proxy_request_body