Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add RobotParser.crawl_delay() and a robots_parsed signal (#7830)

Adrian37661508dbae
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 03996bee6..f7f9f5cca 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -504,6 +504,27 @@ headers_received
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
+robots_parsed
+~~~~~~~~~~~~~
+
+.. signal:: robots_parsed
+.. function:: robots_parsed(robotparser, request)
+
+ .. versionadded:: VERSION
+
+ Sent by
+ :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
+ downloads and parses a :file:`robots.txt` file, for the host that *request*
+ targets.
+
+ This signal supports :ref:`asynchronous handlers <signal-deferred>`.
+
+ :param robotparser: the parser holding the parsed :file:`robots.txt` contents
+ :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
+
+ :param request: the request that triggered the :file:`robots.txt` download
+ :type request: :class:`~scrapy.Request` object
+
Response signals
----------------
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 7d0c17884..81a3a887f 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
+from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
@@ -98,7 +99,7 @@ class RobotsTxtMiddleware:
assert self.crawler.stats
try:
resp = await self.crawler.engine.download_async(robotsreq)
- self._parse_robots(resp, netloc)
+ await self._parse_robots(resp, netloc, request)
except Exception as e:
if not isinstance(e, IgnoreRequest):
logger.error(
@@ -115,13 +116,20 @@ class RobotsTxtMiddleware:
return await maybe_deferred_to_future(parser)
return parser
- def _parse_robots(self, response: Response, netloc: str) -> None:
+ async def _parse_robots(
+ self, response: Response, netloc: str, request: Request
+ ) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
+ await self.crawler.signals.send_catch_log_async(
+ signal=signals.robots_parsed,
+ robotparser=rp,
+ request=request,
+ )
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 0c64ea5a5..b54011784 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta):
:type user_agent: str or bytes
"""
+ def crawl_delay(self, user_agent: str | bytes) -> float | None:
+ """Return the ``Crawl-delay`` directive for ``user_agent`` as a number
+ of seconds, or ``None`` if it is not set or the backend does not support
+ it.
+
+ .. versionadded:: VERSION
+ """
+ return None
+
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser):
url = to_unicode(url)
return self.rp.can_fetch(user_agent, url)
+ def crawl_delay(self, user_agent: str | bytes) -> float | None:
+ delay = self.rp.crawl_delay(to_unicode(user_agent))
+ return None if delay is None else float(delay)
+
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser):
url = to_unicode(url)
return cast("bool", self.rp.is_allowed(user_agent, url))
+ def crawl_delay(self, user_agent: str | bytes) -> float | None:
+ delay = self.rp.get_crawl_delay(to_unicode(user_agent))
+ return None if delay is None else float(delay)
+
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser):
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.can_fetch(url, user_agent)
+
+ def crawl_delay(self, user_agent: str | bytes) -> float | None:
+ delay = self.rp.crawl_delay(to_unicode(user_agent))
+ return None if delay is None else float(delay)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index 972f4fd60..3afeb6eab 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -21,6 +21,7 @@ response_received = object()
response_downloaded = object()
headers_received = object()
bytes_received = object()
+robots_parsed = object()
item_scraped = object()
item_dropped = object()
item_error = object()
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f82041a62..1f2575f8f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -8,6 +8,7 @@ import pytest
from twisted.internet.defer import Deferred, DeferredList
from twisted.python import failure
+from scrapy import signals
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
@@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware:
self.crawler: mock.MagicMock = mock.MagicMock()
self.crawler.settings = Settings()
self.crawler.engine.download_async = mock.AsyncMock()
+ self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])
def teardown_method(self):
del self.crawler
@@ -74,6 +76,21 @@ Disallow: /some/randome/page.html
Request("http://site.local/wiki/Käyttäjä:"), middleware
)
+ @coroutine_test
+ async def test_robotstxt_emits_robots_parsed_signal(self):
+ crawler = self._get_successful_crawler()
+ middleware = RobotsTxtMiddleware(crawler)
+ request = Request("http://site.local/allowed")
+ await self.assertNotIgnored(request, middleware)
+ calls = [
+ kwargs
+ for _, kwargs in crawler.signals.send_catch_log_async.call_args_list
+ if kwargs.get("signal") is signals.robots_parsed
+ ]
+ assert len(calls) == 1
+ assert calls[0]["request"] is request
+ assert calls[0]["robotparser"] is not None
+
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index da94a4e95..ea67877f8 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -4,6 +4,7 @@ from scrapy.robotstxt import (
ProtegoRobotParser,
PythonRobotParser,
RerpRobotParser,
+ RobotParser,
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
@@ -78,6 +79,16 @@ class BaseRobotParserTest:
assert rp.allowed("https://site.local/index.html", "*")
assert rp.allowed("https://site.local/disallowed", "*")
+ def test_crawl_delay(self):
+ robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
+ assert rp.crawl_delay("*") == 10.0
+
+ def test_crawl_delay_unset(self):
+ robotstxt_body = b"User-agent: *\nDisallow: /private\n"
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
+ assert rp.crawl_delay("*") is None
+
def test_unicode_url_and_useragent(self):
robotstxt_robotstxt_body = """
User-Agent: *
@@ -102,6 +113,22 @@ class BaseRobotParserTest:
assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")
+class TestRobotParser:
+ def test_crawl_delay_unsupported(self):
+ class AllowAllRobotParser(RobotParser):
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ return cls()
+
+ def allowed(self, url, user_agent):
+ return True
+
+ rp = AllowAllRobotParser.from_crawler(
+ crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"
+ )
+ assert rp.crawl_delay("*") is None
+
+
class TestDecodeRobotsTxt:
def test_native_string_conversion(self):
robotstxt_body = b"User-agent: *\nDisallow: /\n"