Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove usages of TestCase._wait(). (#6773)

Andrey Rakhmatullin2a1edbd473e4
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 8e718ad5b..408160ccb 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,17 +1,18 @@
+from __future__ import annotations
+
import asyncio
from gzip import BadGzipFile
from unittest import mock
import pytest
-from twisted.internet import defer
-from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
+from twisted.internet.defer import Deferred, succeed
from twisted.trial.unittest import TestCase
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
@@ -29,38 +30,36 @@ class TestManagerBase(TestCase):
def tearDown(self):
return self.crawler.engine.close_spider(self.spider)
- def _download(self, request, response=None):
+ async def _download(
+ self, request: Request, response: Response | None = None
+ ) -> Response | Request:
"""Executes downloader mw manager's download method and returns
- the result (Request or Response) or raise exception in case of
+ the result (Request or Response) or raises exception in case of
failure.
"""
if not response:
response = Response(request.url)
- def download_func(request, spider):
- return response
+ def download_func(request: Request, spider: Spider) -> Deferred[Response]:
+ return succeed(response)
- dfd = self.mwman.download(download_func, request, self.spider)
- # catch deferred result and return the value
- results = []
- dfd.addBoth(results.append)
- self._wait(dfd)
- ret = results[0]
- if isinstance(ret, Failure):
- ret.raiseException()
- return ret
+ return await maybe_deferred_to_future(
+ self.mwman.download(download_func, request, self.spider)
+ )
class TestDefaults(TestManagerBase):
"""Tests default behavior with default settings"""
- def test_request_response(self):
+ @deferred_f_from_coro_f
+ async def test_request_response(self):
req = Request("http://example.com/index.html")
resp = Response(req.url, status=200)
- ret = self._download(req, resp)
+ ret = await self._download(req, resp)
assert isinstance(ret, Response), "Non-response returned"
- def test_3xx_and_invalid_gzipped_body_must_redirect(self):
+ @deferred_f_from_coro_f
+ async def test_3xx_and_invalid_gzipped_body_must_redirect(self):
"""Regression test for a failure when redirecting a compressed
request.
@@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase):
"Location": "http://example.com/login",
},
)
- ret = self._download(request=req, response=resp)
+ ret = await self._download(req, resp)
assert isinstance(ret, Request), f"Not redirected: {ret!r}"
assert to_bytes(ret.url) == resp.headers["Location"], (
"Not redirected to location header"
)
- def test_200_and_invalid_gzipped_body_must_fail(self):
+ @deferred_f_from_coro_f
+ async def test_200_and_invalid_gzipped_body_must_fail(self):
req = Request("http://example.com")
body = b"<p>You are being redirected</p>"
resp = Response(
@@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase):
},
)
with pytest.raises(BadGzipFile):
- self._download(request=req, response=resp)
+ await self._download(req, resp)
class TestResponseFromProcessRequest(TestManagerBase):
"""Tests middleware returning a response from process_request."""
- def test_download_func_not_called(self):
+ @deferred_f_from_coro_f
+ async def test_download_func_not_called(self):
resp = Response("http://example.com/index.html")
class ResponseMiddleware:
@@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase):
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- self._wait(dfd)
-
- assert results[0] is resp
+ result = await maybe_deferred_to_future(
+ self.mwman.download(download_func, req, self.spider)
+ )
+ assert result is resp
assert not download_func.called
-class TestProcessRequestInvalidOutput(TestManagerBase):
- """Invalid return value for process_request method should raise an exception"""
-
- def test_invalid_process_request(self):
+class TestInvalidOutput(TestManagerBase):
+ @deferred_f_from_coro_f
+ async def test_invalid_process_request(self):
+ """Invalid return value for process_request method should raise an exception"""
req = Request("http://example.com/index.html")
class InvalidProcessRequestMiddleware:
@@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase):
return 1
self.mwman._add_middleware(InvalidProcessRequestMiddleware())
- download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- assert isinstance(results[0], Failure)
- assert isinstance(results[0].value, _InvalidOutput)
-
+ with pytest.raises(_InvalidOutput):
+ await self._download(req)
-class TestProcessResponseInvalidOutput(TestManagerBase):
- """Invalid return value for process_response method should raise an exception"""
-
- def test_invalid_process_response(self):
+ @deferred_f_from_coro_f
+ async def test_invalid_process_response(self):
+ """Invalid return value for process_response method should raise an exception"""
req = Request("http://example.com/index.html")
class InvalidProcessResponseMiddleware:
@@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase):
return 1
self.mwman._add_middleware(InvalidProcessResponseMiddleware())
- download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- assert isinstance(results[0], Failure)
- assert isinstance(results[0].value, _InvalidOutput)
+ with pytest.raises(_InvalidOutput):
+ await self._download(req)
-
-class TestProcessExceptionInvalidOutput(TestManagerBase):
- """Invalid return value for process_exception method should raise an exception"""
-
- def test_invalid_process_exception(self):
+ @deferred_f_from_coro_f
+ async def test_invalid_process_exception(self):
+ """Invalid return value for process_exception method should raise an exception"""
req = Request("http://example.com/index.html")
class InvalidProcessExceptionMiddleware:
@@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase):
return 1
self.mwman._add_middleware(InvalidProcessExceptionMiddleware())
- download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- assert isinstance(results[0], Failure)
- assert isinstance(results[0].value, _InvalidOutput)
+ with pytest.raises(_InvalidOutput):
+ await self._download(req)
class TestMiddlewareUsingDeferreds(TestManagerBase):
"""Middlewares using Deferreds should work"""
- def test_deferred(self):
+ @deferred_f_from_coro_f
+ async def test_deferred(self):
resp = Response("http://example.com/index.html")
class DeferredMiddleware:
@@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
self.mwman._add_middleware(DeferredMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- self._wait(dfd)
-
- assert results[0] is resp
+ result = await maybe_deferred_to_future(
+ self.mwman.download(download_func, req, self.spider)
+ )
+ assert result is resp
assert not download_func.called
@@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
class TestMiddlewareUsingCoro(TestManagerBase):
"""Middlewares using asyncio coroutines should work"""
- def test_asyncdef(self):
+ @deferred_f_from_coro_f
+ async def test_asyncdef(self):
resp = Response("http://example.com/index.html")
class CoroMiddleware:
async def process_request(self, request, spider):
- await defer.succeed(42)
+ await succeed(42)
return resp
self.mwman._add_middleware(CoroMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- self._wait(dfd)
-
- assert results[0] is resp
+ result = await maybe_deferred_to_future(
+ self.mwman.download(download_func, req, self.spider)
+ )
+ assert result is resp
assert not download_func.called
@pytest.mark.only_asyncio
- def test_asyncdef_asyncio(self):
+ @deferred_f_from_coro_f
+ async def test_asyncdef_asyncio(self):
resp = Response("http://example.com/index.html")
class CoroMiddleware:
@@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase):
self.mwman._add_middleware(CoroMiddleware())
req = Request("http://example.com/index.html")
download_func = mock.MagicMock()
- dfd = self.mwman.download(download_func, req, self.spider)
- results = []
- dfd.addBoth(results.append)
- self._wait(dfd)
-
- assert results[0] is resp
+ result = await maybe_deferred_to_future(
+ self.mwman.download(download_func, req, self.spider)
+ )
+ assert result is resp
assert not download_func.called
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index ddc9b5206..1d671134e 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,12 +1,12 @@
from __future__ import annotations
from collections.abc import AsyncIterator, Iterable
+from typing import Any
from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from scrapy.core.spidermw import SpiderMiddlewareManager
@@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
-from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
+from scrapy.utils.defer import (
+ deferred_f_from_coro_f,
+ deferred_from_coro,
+ maybe_deferred_to_future,
+)
from scrapy.utils.test import get_crawler
@@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase):
self.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
- def _scrape_response(self):
+ async def _scrape_response(self) -> Any:
"""Execute spider mw manager's scrape_response method and return the result.
Raise exception in case of failure.
"""
scrape_func = mock.MagicMock()
- dfd = self.mwman.scrape_response(
- scrape_func, self.response, self.request, self.spider
+ return await maybe_deferred_to_future(
+ self.mwman.scrape_response(
+ scrape_func, self.response, self.request, self.spider
+ )
)
- # catch deferred result and return the value
- results = []
- dfd.addBoth(results.append)
- self._wait(dfd)
- return results[0]
class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_input method"""
- def test_invalid_process_spider_input(self):
+ @deferred_f_from_coro_f
+ async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
return 1
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
- result = self._scrape_response()
- assert isinstance(result, Failure)
- assert isinstance(result.value, _InvalidOutput)
+ with pytest.raises(_InvalidOutput):
+ await self._scrape_response()
class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_output method"""
- def test_invalid_process_spider_output(self):
+ @deferred_f_from_coro_f
+ async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider):
return 1
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
- result = self._scrape_response()
- assert isinstance(result, Failure)
- assert isinstance(result.value, _InvalidOutput)
+ with pytest.raises(_InvalidOutput):
+ await self._scrape_response()
class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
"""Invalid return value for process_spider_exception method"""
- def test_invalid_process_spider_exception(self):
+ @deferred_f_from_coro_f
+ async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception, spider):
return 1
@@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
- result = self._scrape_response()
- assert isinstance(result, Failure)
- assert isinstance(result.value, _InvalidOutput)
+ with pytest.raises(_InvalidOutput):
+ await self._scrape_response()
class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
"""Re raise the exception by returning None"""
- def test_process_spider_exception_return_none(self):
+ @deferred_f_from_coro_f
+ async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception, spider):
return None
@@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
- result = self._scrape_response()
- assert isinstance(result, Failure)
- assert isinstance(result.value, ZeroDivisionError)
+ with pytest.raises(ZeroDivisionError):
+ await self._scrape_response()
class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):