Scrapy, a fast high-level web crawling & scraping framework for Python.
Convert test_downloadermiddleware_robotstxt.py from callbacks to awaits. (#6743)
Andrey Rakhmatullin9f99da8f865e
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex ad335f852..9518f1835 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -1,9 +1,11 @@-from typing import Any+from __future__ import annotations++from typing import TYPE_CHECKINGfrom unittest import mockimport pytestfrom twisted.internet import error, reactor-from twisted.internet.defer import Deferred, DeferredList, maybeDeferred+from twisted.internet.defer import Deferred, maybeDeferredfrom twisted.python import failurefrom twisted.trial import unittest@@ -13,8 +15,12 @@ from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.settings import Settings+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom tests.test_robotstxt_interface import rerp_available+if TYPE_CHECKING:+ from scrapy.crawler import Crawler+class TestRobotsTxtMiddleware(unittest.TestCase):def setUp(self):@@ -31,7 +37,7 @@ class TestRobotsTxtMiddleware(unittest.TestCase):with pytest.raises(NotConfigured):RobotsTxtMiddleware(self.crawler)- def _get_successful_crawler(self):+ def _get_successful_crawler(self) -> Crawler:crawler = self.crawlercrawler.settings.set("ROBOTSTXT_OBEY", True)ROBOTS = """@@ -54,54 +60,41 @@ Disallow: /some/randome/page.htmlcrawler.engine.download.side_effect = return_responsereturn crawler- def test_robotstxt(self):+ @deferred_f_from_coro_f+ async def test_robotstxt(self):middleware = RobotsTxtMiddleware(self._get_successful_crawler())- return DeferredList(- [- self.assertNotIgnored(Request("http://site.local/allowed"), middleware),- maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"),- self.assertIgnored(Request("http://site.local/admin/main"), middleware),- self.assertIgnored(Request("http://site.local/static/"), middleware),- self.assertIgnored(- Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"),- middleware,- ),- self.assertIgnored(- Request("http://site.local/wiki/Käyttäjä:"), middleware- ),- ],- fireOnOneErrback=True,+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)+ self.assertRobotsTxtRequested("http://site.local")+ await self.assertIgnored(Request("http://site.local/admin/main"), middleware)+ await self.assertIgnored(Request("http://site.local/static/"), middleware)+ await self.assertIgnored(+ Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware+ )+ await self.assertIgnored(+ Request("http://site.local/wiki/Käyttäjä:"), middleware)- def test_robotstxt_ready_parser(self):+ @deferred_f_from_coro_f+ async def test_robotstxt_ready_parser(self):middleware = RobotsTxtMiddleware(self._get_successful_crawler())- d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)- d.addCallback(- lambda _: self.assertNotIgnored(- Request("http://site.local/allowed"), middleware- )- )- return d+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)- def test_robotstxt_meta(self):+ @deferred_f_from_coro_f+ async def test_robotstxt_meta(self):middleware = RobotsTxtMiddleware(self._get_successful_crawler())meta = {"dont_obey_robotstxt": True}- return DeferredList(- [- self.assertNotIgnored(- Request("http://site.local/allowed", meta=meta), middleware- ),- self.assertNotIgnored(- Request("http://site.local/admin/main", meta=meta), middleware- ),- self.assertNotIgnored(- Request("http://site.local/static/", meta=meta), middleware- ),- ],- fireOnOneErrback=True,+ await self.assertNotIgnored(+ Request("http://site.local/allowed", meta=meta), middleware+ )+ await self.assertNotIgnored(+ Request("http://site.local/admin/main", meta=meta), middleware+ )+ await self.assertNotIgnored(+ Request("http://site.local/static/", meta=meta), middleware)- def _get_garbage_crawler(self):+ def _get_garbage_crawler(self) -> Crawler:crawler = self.crawlercrawler.settings.set("ROBOTSTXT_OBEY", True)response = Response(@@ -116,22 +109,16 @@ Disallow: /some/randome/page.htmlcrawler.engine.download.side_effect = return_responsereturn crawler- def test_robotstxt_garbage(self):+ @deferred_f_from_coro_f+ async def test_robotstxt_garbage(self):# garbage response should be discarded, equal 'allow all'middleware = RobotsTxtMiddleware(self._get_garbage_crawler())- return DeferredList(- [- self.assertNotIgnored(Request("http://site.local"), middleware),- self.assertNotIgnored(Request("http://site.local/allowed"), middleware),- self.assertNotIgnored(- Request("http://site.local/admin/main"), middleware- ),- self.assertNotIgnored(Request("http://site.local/static/"), middleware),- ],- fireOnOneErrback=True,- )+ await self.assertNotIgnored(Request("http://site.local"), middleware)+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)+ await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)+ await self.assertNotIgnored(Request("http://site.local/static/"), middleware)- def _get_emptybody_crawler(self):+ def _get_emptybody_crawler(self) -> Crawler:crawler = self.crawlercrawler.settings.set("ROBOTSTXT_OBEY", True)response = Response("http://site.local/robots.txt")@@ -144,21 +131,16 @@ Disallow: /some/randome/page.htmlcrawler.engine.download.side_effect = return_responsereturn crawler- def test_robotstxt_empty_response(self):+ @deferred_f_from_coro_f+ async def test_robotstxt_empty_response(self):# empty response should equal 'allow all'middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())- return DeferredList(- [- self.assertNotIgnored(Request("http://site.local/allowed"), middleware),- self.assertNotIgnored(- Request("http://site.local/admin/main"), middleware- ),- self.assertNotIgnored(Request("http://site.local/static/"), middleware),- ],- fireOnOneErrback=True,- )+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)+ await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)+ await self.assertNotIgnored(Request("http://site.local/static/"), middleware)- def test_robotstxt_error(self):+ @deferred_f_from_coro_f+ async def test_robotstxt_error(self):self.crawler.settings.set("ROBOTSTXT_OBEY", True)err = error.DNSLookupError("Robotstxt address not found")@@ -171,15 +153,13 @@ Disallow: /some/randome/page.htmlmiddleware = RobotsTxtMiddleware(self.crawler)middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)- deferred = middleware.process_request(Request("http://site.local"), None)-- def check_called(_: Any) -> None:- assert middleware._logerror.called-- deferred.addCallback(check_called)- return deferred+ await maybe_deferred_to_future(+ middleware.process_request(Request("http://site.local"), None)+ )+ assert middleware._logerror.called- def test_robotstxt_immediate_error(self):+ @deferred_f_from_coro_f+ async def test_robotstxt_immediate_error(self):self.crawler.settings.set("ROBOTSTXT_OBEY", True)err = error.DNSLookupError("Robotstxt address not found")@@ -191,9 +171,10 @@ Disallow: /some/randome/page.htmlself.crawler.engine.download.side_effect = immediate_failuremiddleware = RobotsTxtMiddleware(self.crawler)- return self.assertNotIgnored(Request("http://site.local"), middleware)+ await self.assertNotIgnored(Request("http://site.local"), middleware)- def test_ignore_robotstxt_request(self):+ @deferred_f_from_coro_f+ async def test_ignore_robotstxt_request(self):self.crawler.settings.set("ROBOTSTXT_OBEY", True)def ignore_request(request):@@ -206,13 +187,8 @@ Disallow: /some/randome/page.htmlmiddleware = RobotsTxtMiddleware(self.crawler)mw_module_logger.error = mock.MagicMock()- d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)-- def check_not_called(_: Any) -> None:- assert not mw_module_logger.error.called # type: ignore[attr-defined]-- d.addCallback(check_not_called)- return d+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)+ assert not mw_module_logger.error.called # type: ignore[attr-defined]def test_robotstxt_user_agent_setting(self):crawler = self._get_successful_crawler()@@ -236,19 +212,27 @@ Disallow: /some/randome/page.htmlDeferred,)- def assertNotIgnored(self, request, middleware):+ async def assertNotIgnored(+ self, request: Request, middleware: RobotsTxtMiddleware+ ) -> None:spider = None # not actually used- dfd = maybeDeferred(middleware.process_request, request, spider)- dfd.addCallback(self.assertIsNone)- return dfd+ result = await maybe_deferred_to_future(+ maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload]+ )+ assert result is None- def assertIgnored(self, request, middleware):+ async def assertIgnored(+ self, request: Request, middleware: RobotsTxtMiddleware+ ) -> None:spider = None # not actually used- return self.assertFailure(- maybeDeferred(middleware.process_request, request, spider), IgnoreRequest+ await maybe_deferred_to_future(+ self.assertFailure(+ middleware.process_request(request, spider), # type: ignore[arg-type]+ IgnoreRequest,+ ))- def assertRobotsTxtRequested(self, base_url):+ def assertRobotsTxtRequested(self, base_url: str) -> None:calls = self.crawler.engine.download.call_args_listrequest = calls[0][0][0]assert request.url == f"{base_url}/robots.txt"