Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove usages of TestCase._wait(). (#6773)
Andrey Rakhmatullin2a1edbd473e4
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 8e718ad5b..408160ccb 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -1,17 +1,18 @@+from __future__ import annotations+import asynciofrom gzip import BadGzipFilefrom unittest import mockimport pytest-from twisted.internet import defer-from twisted.internet.defer import Deferred-from twisted.python.failure import Failure+from twisted.internet.defer import Deferred, succeedfrom twisted.trial.unittest import TestCasefrom scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.exceptions import _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spider+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler, get_from_asyncio_queue@@ -29,38 +30,36 @@ class TestManagerBase(TestCase):def tearDown(self):return self.crawler.engine.close_spider(self.spider)- def _download(self, request, response=None):+ async def _download(+ self, request: Request, response: Response | None = None+ ) -> Response | Request:"""Executes downloader mw manager's download method and returns- the result (Request or Response) or raise exception in case of+ the result (Request or Response) or raises exception in case offailure."""if not response:response = Response(request.url)- def download_func(request, spider):- return response+ def download_func(request: Request, spider: Spider) -> Deferred[Response]:+ return succeed(response)- dfd = self.mwman.download(download_func, request, self.spider)- # catch deferred result and return the value- results = []- dfd.addBoth(results.append)- self._wait(dfd)- ret = results[0]- if isinstance(ret, Failure):- ret.raiseException()- return ret+ return await maybe_deferred_to_future(+ self.mwman.download(download_func, request, self.spider)+ )class TestDefaults(TestManagerBase):"""Tests default behavior with default settings"""- def test_request_response(self):+ @deferred_f_from_coro_f+ async def test_request_response(self):req = Request("http://example.com/index.html")resp = Response(req.url, status=200)- ret = self._download(req, resp)+ ret = await self._download(req, resp)assert isinstance(ret, Response), "Non-response returned"- def test_3xx_and_invalid_gzipped_body_must_redirect(self):+ @deferred_f_from_coro_f+ async def test_3xx_and_invalid_gzipped_body_must_redirect(self):"""Regression test for a failure when redirecting a compressedrequest.@@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase):"Location": "http://example.com/login",},)- ret = self._download(request=req, response=resp)+ ret = await self._download(req, resp)assert isinstance(ret, Request), f"Not redirected: {ret!r}"assert to_bytes(ret.url) == resp.headers["Location"], ("Not redirected to location header")- def test_200_and_invalid_gzipped_body_must_fail(self):+ @deferred_f_from_coro_f+ async def test_200_and_invalid_gzipped_body_must_fail(self):req = Request("http://example.com")body = b"<p>You are being redirected</p>"resp = Response(@@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase):},)with pytest.raises(BadGzipFile):- self._download(request=req, response=resp)+ await self._download(req, resp)class TestResponseFromProcessRequest(TestManagerBase):"""Tests middleware returning a response from process_request."""- def test_download_func_not_called(self):+ @deferred_f_from_coro_f+ async def test_download_func_not_called(self):resp = Response("http://example.com/index.html")class ResponseMiddleware:@@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase):req = Request("http://example.com/index.html")download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- self._wait(dfd)-- assert results[0] is resp+ result = await maybe_deferred_to_future(+ self.mwman.download(download_func, req, self.spider)+ )+ assert result is respassert not download_func.called-class TestProcessRequestInvalidOutput(TestManagerBase):- """Invalid return value for process_request method should raise an exception"""-- def test_invalid_process_request(self):+class TestInvalidOutput(TestManagerBase):+ @deferred_f_from_coro_f+ async def test_invalid_process_request(self):+ """Invalid return value for process_request method should raise an exception"""req = Request("http://example.com/index.html")class InvalidProcessRequestMiddleware:@@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase):return 1self.mwman._add_middleware(InvalidProcessRequestMiddleware())- download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- assert isinstance(results[0], Failure)- assert isinstance(results[0].value, _InvalidOutput)-+ with pytest.raises(_InvalidOutput):+ await self._download(req)-class TestProcessResponseInvalidOutput(TestManagerBase):- """Invalid return value for process_response method should raise an exception"""-- def test_invalid_process_response(self):+ @deferred_f_from_coro_f+ async def test_invalid_process_response(self):+ """Invalid return value for process_response method should raise an exception"""req = Request("http://example.com/index.html")class InvalidProcessResponseMiddleware:@@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase):return 1self.mwman._add_middleware(InvalidProcessResponseMiddleware())- download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- assert isinstance(results[0], Failure)- assert isinstance(results[0].value, _InvalidOutput)+ with pytest.raises(_InvalidOutput):+ await self._download(req)--class TestProcessExceptionInvalidOutput(TestManagerBase):- """Invalid return value for process_exception method should raise an exception"""-- def test_invalid_process_exception(self):+ @deferred_f_from_coro_f+ async def test_invalid_process_exception(self):+ """Invalid return value for process_exception method should raise an exception"""req = Request("http://example.com/index.html")class InvalidProcessExceptionMiddleware:@@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase):return 1self.mwman._add_middleware(InvalidProcessExceptionMiddleware())- download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- assert isinstance(results[0], Failure)- assert isinstance(results[0].value, _InvalidOutput)+ with pytest.raises(_InvalidOutput):+ await self._download(req)class TestMiddlewareUsingDeferreds(TestManagerBase):"""Middlewares using Deferreds should work"""- def test_deferred(self):+ @deferred_f_from_coro_f+ async def test_deferred(self):resp = Response("http://example.com/index.html")class DeferredMiddleware:@@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):self.mwman._add_middleware(DeferredMiddleware())req = Request("http://example.com/index.html")download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- self._wait(dfd)-- assert results[0] is resp+ result = await maybe_deferred_to_future(+ self.mwman.download(download_func, req, self.spider)+ )+ assert result is respassert not download_func.called@@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):class TestMiddlewareUsingCoro(TestManagerBase):"""Middlewares using asyncio coroutines should work"""- def test_asyncdef(self):+ @deferred_f_from_coro_f+ async def test_asyncdef(self):resp = Response("http://example.com/index.html")class CoroMiddleware:async def process_request(self, request, spider):- await defer.succeed(42)+ await succeed(42)return respself.mwman._add_middleware(CoroMiddleware())req = Request("http://example.com/index.html")download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- self._wait(dfd)-- assert results[0] is resp+ result = await maybe_deferred_to_future(+ self.mwman.download(download_func, req, self.spider)+ )+ assert result is respassert not download_func.called@pytest.mark.only_asyncio- def test_asyncdef_asyncio(self):+ @deferred_f_from_coro_f+ async def test_asyncdef_asyncio(self):resp = Response("http://example.com/index.html")class CoroMiddleware:@@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase):self.mwman._add_middleware(CoroMiddleware())req = Request("http://example.com/index.html")download_func = mock.MagicMock()- dfd = self.mwman.download(download_func, req, self.spider)- results = []- dfd.addBoth(results.append)- self._wait(dfd)-- assert results[0] is resp+ result = await maybe_deferred_to_future(+ self.mwman.download(download_func, req, self.spider)+ )+ assert result is respassert not download_func.calleddiff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex ddc9b5206..1d671134e 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -1,12 +1,12 @@from __future__ import annotationsfrom collections.abc import AsyncIterator, Iterable+from typing import Anyfrom unittest import mockimport pytestfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.python.failure import Failurefrom twisted.trial.unittest import TestCasefrom scrapy.core.spidermw import SpiderMiddlewareManager@@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.asyncgen import collect_asyncgen-from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future+from scrapy.utils.defer import (+ deferred_f_from_coro_f,+ deferred_from_coro,+ maybe_deferred_to_future,+)from scrapy.utils.test import get_crawler@@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase):self.spider = self.crawler._create_spider("foo")self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)- def _scrape_response(self):+ async def _scrape_response(self) -> Any:"""Execute spider mw manager's scrape_response method and return the result.Raise exception in case of failure."""scrape_func = mock.MagicMock()- dfd = self.mwman.scrape_response(- scrape_func, self.response, self.request, self.spider+ return await maybe_deferred_to_future(+ self.mwman.scrape_response(+ scrape_func, self.response, self.request, self.spider+ ))- # catch deferred result and return the value- results = []- dfd.addBoth(results.append)- self._wait(dfd)- return results[0]class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):"""Invalid return value for process_spider_input method"""- def test_invalid_process_spider_input(self):+ @deferred_f_from_coro_f+ async def test_invalid_process_spider_input(self):class InvalidProcessSpiderInputMiddleware:def process_spider_input(self, response, spider):return 1self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())- result = self._scrape_response()- assert isinstance(result, Failure)- assert isinstance(result.value, _InvalidOutput)+ with pytest.raises(_InvalidOutput):+ await self._scrape_response()class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):"""Invalid return value for process_spider_output method"""- def test_invalid_process_spider_output(self):+ @deferred_f_from_coro_f+ async def test_invalid_process_spider_output(self):class InvalidProcessSpiderOutputMiddleware:def process_spider_output(self, response, result, spider):return 1self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())- result = self._scrape_response()- assert isinstance(result, Failure)- assert isinstance(result.value, _InvalidOutput)+ with pytest.raises(_InvalidOutput):+ await self._scrape_response()class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):"""Invalid return value for process_spider_exception method"""- def test_invalid_process_spider_exception(self):+ @deferred_f_from_coro_f+ async def test_invalid_process_spider_exception(self):class InvalidProcessSpiderOutputExceptionMiddleware:def process_spider_exception(self, response, exception, spider):return 1@@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())- result = self._scrape_response()- assert isinstance(result, Failure)- assert isinstance(result.value, _InvalidOutput)+ with pytest.raises(_InvalidOutput):+ await self._scrape_response()class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):"""Re raise the exception by returning None"""- def test_process_spider_exception_return_none(self):+ @deferred_f_from_coro_f+ async def test_process_spider_exception_return_none(self):class ProcessSpiderExceptionReturnNoneMiddleware:def process_spider_exception(self, response, exception, spider):return None@@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())- result = self._scrape_response()- assert isinstance(result, Failure)- assert isinstance(result.value, ZeroDivisionError)+ with pytest.raises(ZeroDivisionError):+ await self._scrape_response()class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):