Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Convert test_downloadermiddleware_robotstxt.py from callbacks to awaits. (#6743)

Andrey Rakhmatullin9f99da8f865e
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index ad335f852..9518f1835 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,9 +1,11 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
from unittest import mock
import pytest
from twisted.internet import error, reactor
-from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
+from twisted.internet.defer import Deferred, maybeDeferred
from twisted.python import failure
from twisted.trial import unittest
@@ -13,8 +15,12 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from tests.test_robotstxt_interface import rerp_available
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+
class TestRobotsTxtMiddleware(unittest.TestCase):
def setUp(self):
@@ -31,7 +37,7 @@ class TestRobotsTxtMiddleware(unittest.TestCase):
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
- def _get_successful_crawler(self):
+ def _get_successful_crawler(self) -> Crawler:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
ROBOTS = """
@@ -54,54 +60,41 @@ Disallow: /some/randome/page.html
crawler.engine.download.side_effect = return_response
return crawler
- def test_robotstxt(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
- return DeferredList(
- [
- self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
- maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"),
- self.assertIgnored(Request("http://site.local/admin/main"), middleware),
- self.assertIgnored(Request("http://site.local/static/"), middleware),
- self.assertIgnored(
- Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"),
- middleware,
- ),
- self.assertIgnored(
- Request("http://site.local/wiki/Käyttäjä:"), middleware
- ),
- ],
- fireOnOneErrback=True,
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+ self.assertRobotsTxtRequested("http://site.local")
+ await self.assertIgnored(Request("http://site.local/admin/main"), middleware)
+ await self.assertIgnored(Request("http://site.local/static/"), middleware)
+ await self.assertIgnored(
+ Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware
+ )
+ await self.assertIgnored(
+ Request("http://site.local/wiki/Käyttäjä:"), middleware
)
- def test_robotstxt_ready_parser(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt_ready_parser(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
- d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
- d.addCallback(
- lambda _: self.assertNotIgnored(
- Request("http://site.local/allowed"), middleware
- )
- )
- return d
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
- def test_robotstxt_meta(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt_meta(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
meta = {"dont_obey_robotstxt": True}
- return DeferredList(
- [
- self.assertNotIgnored(
- Request("http://site.local/allowed", meta=meta), middleware
- ),
- self.assertNotIgnored(
- Request("http://site.local/admin/main", meta=meta), middleware
- ),
- self.assertNotIgnored(
- Request("http://site.local/static/", meta=meta), middleware
- ),
- ],
- fireOnOneErrback=True,
+ await self.assertNotIgnored(
+ Request("http://site.local/allowed", meta=meta), middleware
+ )
+ await self.assertNotIgnored(
+ Request("http://site.local/admin/main", meta=meta), middleware
+ )
+ await self.assertNotIgnored(
+ Request("http://site.local/static/", meta=meta), middleware
)
- def _get_garbage_crawler(self):
+ def _get_garbage_crawler(self) -> Crawler:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response(
@@ -116,22 +109,16 @@ Disallow: /some/randome/page.html
crawler.engine.download.side_effect = return_response
return crawler
- def test_robotstxt_garbage(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt_garbage(self):
# garbage response should be discarded, equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
- return DeferredList(
- [
- self.assertNotIgnored(Request("http://site.local"), middleware),
- self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
- self.assertNotIgnored(
- Request("http://site.local/admin/main"), middleware
- ),
- self.assertNotIgnored(Request("http://site.local/static/"), middleware),
- ],
- fireOnOneErrback=True,
- )
+ await self.assertNotIgnored(Request("http://site.local"), middleware)
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+ await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
+ await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
- def _get_emptybody_crawler(self):
+ def _get_emptybody_crawler(self) -> Crawler:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response("http://site.local/robots.txt")
@@ -144,21 +131,16 @@ Disallow: /some/randome/page.html
crawler.engine.download.side_effect = return_response
return crawler
- def test_robotstxt_empty_response(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt_empty_response(self):
# empty response should equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
- return DeferredList(
- [
- self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
- self.assertNotIgnored(
- Request("http://site.local/admin/main"), middleware
- ),
- self.assertNotIgnored(Request("http://site.local/static/"), middleware),
- ],
- fireOnOneErrback=True,
- )
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+ await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
+ await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
- def test_robotstxt_error(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt_error(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = error.DNSLookupError("Robotstxt address not found")
@@ -171,15 +153,13 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
- deferred = middleware.process_request(Request("http://site.local"), None)
-
- def check_called(_: Any) -> None:
- assert middleware._logerror.called
-
- deferred.addCallback(check_called)
- return deferred
+ await maybe_deferred_to_future(
+ middleware.process_request(Request("http://site.local"), None)
+ )
+ assert middleware._logerror.called
- def test_robotstxt_immediate_error(self):
+ @deferred_f_from_coro_f
+ async def test_robotstxt_immediate_error(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = error.DNSLookupError("Robotstxt address not found")
@@ -191,9 +171,10 @@ Disallow: /some/randome/page.html
self.crawler.engine.download.side_effect = immediate_failure
middleware = RobotsTxtMiddleware(self.crawler)
- return self.assertNotIgnored(Request("http://site.local"), middleware)
+ await self.assertNotIgnored(Request("http://site.local"), middleware)
- def test_ignore_robotstxt_request(self):
+ @deferred_f_from_coro_f
+ async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
def ignore_request(request):
@@ -206,13 +187,8 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self.crawler)
mw_module_logger.error = mock.MagicMock()
- d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
-
- def check_not_called(_: Any) -> None:
- assert not mw_module_logger.error.called # type: ignore[attr-defined]
-
- d.addCallback(check_not_called)
- return d
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+ assert not mw_module_logger.error.called # type: ignore[attr-defined]
def test_robotstxt_user_agent_setting(self):
crawler = self._get_successful_crawler()
@@ -236,19 +212,27 @@ Disallow: /some/randome/page.html
Deferred,
)
- def assertNotIgnored(self, request, middleware):
+ async def assertNotIgnored(
+ self, request: Request, middleware: RobotsTxtMiddleware
+ ) -> None:
spider = None # not actually used
- dfd = maybeDeferred(middleware.process_request, request, spider)
- dfd.addCallback(self.assertIsNone)
- return dfd
+ result = await maybe_deferred_to_future(
+ maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload]
+ )
+ assert result is None
- def assertIgnored(self, request, middleware):
+ async def assertIgnored(
+ self, request: Request, middleware: RobotsTxtMiddleware
+ ) -> None:
spider = None # not actually used
- return self.assertFailure(
- maybeDeferred(middleware.process_request, request, spider), IgnoreRequest
+ await maybe_deferred_to_future(
+ self.assertFailure(
+ middleware.process_request(request, spider), # type: ignore[arg-type]
+ IgnoreRequest,
+ )
)
- def assertRobotsTxtRequested(self, base_url):
+ def assertRobotsTxtRequested(self, base_url: str) -> None:
calls = self.crawler.engine.download.call_args_list
request = calls[0][0][0]
assert request.url == f"{base_url}/robots.txt"