Scrapy, a fast high-level web crawling & scraping framework for Python.
Add RobotParser.crawl_delay() and a robots_parsed signal (#7830)
Adrian37661508dbae
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex 03996bee6..f7f9f5cca 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -504,6 +504,27 @@ headers_received:param spider: the spider associated with the response:type spider: :class:`~scrapy.Spider` object+robots_parsed+~~~~~~~~~~~~~++.. signal:: robots_parsed+.. function:: robots_parsed(robotparser, request)++ .. versionadded:: VERSION++ Sent by+ :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it+ downloads and parses a :file:`robots.txt` file, for the host that *request*+ targets.++ This signal supports :ref:`asynchronous handlers <signal-deferred>`.++ :param robotparser: the parser holding the parsed :file:`robots.txt` contents+ :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object++ :param request: the request that triggered the :file:`robots.txt` download+ :type request: :class:`~scrapy.Request` object+Response signals----------------diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 7d0c17884..81a3a887f 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -11,6 +11,7 @@ from typing import TYPE_CHECKINGfrom twisted.internet.defer import Deferred+from scrapy import signalsfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK@@ -98,7 +99,7 @@ class RobotsTxtMiddleware:assert self.crawler.statstry:resp = await self.crawler.engine.download_async(robotsreq)- self._parse_robots(resp, netloc)+ await self._parse_robots(resp, netloc, request)except Exception as e:if not isinstance(e, IgnoreRequest):logger.error(@@ -115,13 +116,20 @@ class RobotsTxtMiddleware:return await maybe_deferred_to_future(parser)return parser- def _parse_robots(self, response: Response, netloc: str) -> None:+ async def _parse_robots(+ self, response: Response, netloc: str, request: Request+ ) -> None:assert self.crawler.statsself.crawler.stats.inc_value("robotstxt/response_count")self.crawler.stats.inc_value(f"robotstxt/response_status_count/{response.status}")rp = self._parserimpl.from_crawler(self.crawler, response.body)+ await self.crawler.signals.send_catch_log_async(+ signal=signals.robots_parsed,+ robotparser=rp,+ request=request,+ )rp_dfd = self._parsers[netloc]assert isinstance(rp_dfd, Deferred)self._parsers[netloc] = rpdiff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 0c64ea5a5..b54011784 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta)::type user_agent: str or bytes"""+ def crawl_delay(self, user_agent: str | bytes) -> float | None:+ """Return the ``Crawl-delay`` directive for ``user_agent`` as a number+ of seconds, or ``None`` if it is not set or the backend does not support+ it.++ .. versionadded:: VERSION+ """+ return None+class PythonRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):@@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser):url = to_unicode(url)return self.rp.can_fetch(user_agent, url)+ def crawl_delay(self, user_agent: str | bytes) -> float | None:+ delay = self.rp.crawl_delay(to_unicode(user_agent))+ return None if delay is None else float(delay)+class RerpRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):@@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser):url = to_unicode(url)return cast("bool", self.rp.is_allowed(user_agent, url))+ def crawl_delay(self, user_agent: str | bytes) -> float | None:+ delay = self.rp.get_crawl_delay(to_unicode(user_agent))+ return None if delay is None else float(delay)+class ProtegoRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):@@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser):user_agent = to_unicode(user_agent)url = to_unicode(url)return self.rp.can_fetch(url, user_agent)++ def crawl_delay(self, user_agent: str | bytes) -> float | None:+ delay = self.rp.crawl_delay(to_unicode(user_agent))+ return None if delay is None else float(delay)diff --git a/scrapy/signals.py b/scrapy/signals.pyindex 972f4fd60..3afeb6eab 100644--- a/scrapy/signals.py+++ b/scrapy/signals.py@@ -21,6 +21,7 @@ response_received = object()response_downloaded = object()headers_received = object()bytes_received = object()+robots_parsed = object()item_scraped = object()item_dropped = object()item_error = object()diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex f82041a62..1f2575f8f 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -8,6 +8,7 @@ import pytestfrom twisted.internet.defer import Deferred, DeferredListfrom twisted.python import failure+from scrapy import signalsfrom scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddlewarefrom scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponse@@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware:self.crawler: mock.MagicMock = mock.MagicMock()self.crawler.settings = Settings()self.crawler.engine.download_async = mock.AsyncMock()+ self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])def teardown_method(self):del self.crawler@@ -74,6 +76,21 @@ Disallow: /some/randome/page.htmlRequest("http://site.local/wiki/Käyttäjä:"), middleware)+ @coroutine_test+ async def test_robotstxt_emits_robots_parsed_signal(self):+ crawler = self._get_successful_crawler()+ middleware = RobotsTxtMiddleware(crawler)+ request = Request("http://site.local/allowed")+ await self.assertNotIgnored(request, middleware)+ calls = [+ kwargs+ for _, kwargs in crawler.signals.send_catch_log_async.call_args_list+ if kwargs.get("signal") is signals.robots_parsed+ ]+ assert len(calls) == 1+ assert calls[0]["request"] is request+ assert calls[0]["robotparser"] is not None+@coroutine_testasync def test_robotstxt_multiple_reqs(self) -> None:middleware = RobotsTxtMiddleware(self._get_successful_crawler())diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex da94a4e95..ea67877f8 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -4,6 +4,7 @@ from scrapy.robotstxt import (ProtegoRobotParser,PythonRobotParser,RerpRobotParser,+ RobotParser,decode_robotstxt,)from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER@@ -78,6 +79,16 @@ class BaseRobotParserTest:assert rp.allowed("https://site.local/index.html", "*")assert rp.allowed("https://site.local/disallowed", "*")+ def test_crawl_delay(self):+ robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)+ assert rp.crawl_delay("*") == 10.0++ def test_crawl_delay_unset(self):+ robotstxt_body = b"User-agent: *\nDisallow: /private\n"+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)+ assert rp.crawl_delay("*") is None+def test_unicode_url_and_useragent(self):robotstxt_robotstxt_body = """User-Agent: *@@ -102,6 +113,22 @@ class BaseRobotParserTest:assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")+class TestRobotParser:+ def test_crawl_delay_unsupported(self):+ class AllowAllRobotParser(RobotParser):+ @classmethod+ def from_crawler(cls, crawler, robotstxt_body):+ return cls()++ def allowed(self, url, user_agent):+ return True++ rp = AllowAllRobotParser.from_crawler(+ crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"+ )+ assert rp.crawl_delay("*") is None++class TestDecodeRobotsTxt:def test_native_string_conversion(self):robotstxt_body = b"User-agent: *\nDisallow: /\n"