Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

adds ROBOTSTXT_USER_AGENT setting

Anubhav Patel00fe05e53611
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 616b56101..ae413dc84 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1074,6 +1074,21 @@ implementing the methods described below.
.. autoclass:: RobotParser
:members:
+RobotsTxtMiddleware Settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt_ file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt_ file.
+
.. _robots.txt: http://www.robotstxt.org/
DownloaderStats
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 12606fe47..c4b55cc7b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1151,6 +1151,18 @@ Default: ``'scrapy.robotstxt.PythonRobotParser'``
The parser backend to use for parsing ``robots.txt`` files. For more information see
:ref:`topics-dlmw-robots`.
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt file.
+
.. setting:: SCHEDULER
SCHEDULER
@@ -1409,7 +1421,9 @@ USER_AGENT
Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
-The default User-Agent to use when crawling, unless overridden.
+The default User-Agent to use when crawling, unless overridden. This user agent is
+also used in robots.txt if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+there is no overridding User-Agent header specified for the request.
Settings documented elsewhere:
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c5a60d355..6a5dfb79c 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -26,6 +26,7 @@ class RobotsTxtMiddleware(object):
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
raise NotConfigured
self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
+ self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
self.crawler = crawler
self._parsers = {}
self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
@@ -47,7 +48,10 @@ class RobotsTxtMiddleware(object):
def process_request_2(self, rp, request, spider):
if rp is None:
return
- useragent = request.headers.get(b'User-Agent', self._default_useragent)
+
+ useragent = self._robotstxt_useragent
+ if not useragent:
+ useragent = request.headers.get(b'User-Agent', self._default_useragent)
if not rp.allowed(request.url, useragent):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 81fee543f..efb257e2b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -246,6 +246,7 @@ RETRY_PRIORITY_ADJUST = -1
ROBOTSTXT_OBEY = False
ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
+ROBOTSTXT_USER_AGENT = None
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 79f172848..fbc46cba4 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -164,6 +164,15 @@ Disallow: /some/randome/page.html
d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
return d
+ def test_robotstxt_user_agent_setting(self):
+ crawler = self._get_successful_crawler()
+ crawler.settings.set('ROBOTSTXT_USER_AGENT', 'Examplebot')
+ crawler.settings.set('USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64)')
+ middleware = RobotsTxtMiddleware(crawler)
+ rp = mock.MagicMock(return_value=True)
+ middleware.process_request_2(rp, Request('http://site.local/allowed'), None)
+ rp.allowed.assert_called_once_with('http://site.local/allowed', 'Examplebot')
+
def assertNotIgnored(self, request, middleware):
spider = None # not actually used
dfd = maybeDeferred(middleware.process_request, request, spider)