Scrapy, a fast high-level web crawling & scraping framework for Python.
adds ROBOTSTXT_USER_AGENT setting
Anubhav Patel00fe05e53611
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 616b56101..ae413dc84 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -1074,6 +1074,21 @@ implementing the methods described below... autoclass:: RobotParser:members:+RobotsTxtMiddleware Settings+~~~~~~~~~~~~~~~~~~~~~~~~~~~~++.. setting:: ROBOTSTXT_USER_AGENT++ROBOTSTXT_USER_AGENT+^^^^^^^^^^^^^^^^^^^^++Default: ``None``++The user agent string to use for matching in the robots.txt_ file. If ``None``,+the User-Agent header you are sending with the request or the+:setting:`USER_AGENT` setting (in that order) will be used for determining+the user agent to use in the robots.txt_ file.+.. _robots.txt: http://www.robotstxt.org/DownloaderStatsdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 12606fe47..c4b55cc7b 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1151,6 +1151,18 @@ Default: ``'scrapy.robotstxt.PythonRobotParser'``The parser backend to use for parsing ``robots.txt`` files. For more information see:ref:`topics-dlmw-robots`.+.. setting:: ROBOTSTXT_USER_AGENT++ROBOTSTXT_USER_AGENT+^^^^^^^^^^^^^^^^^^^^++Default: ``None``++The user agent string to use for matching in the robots.txt file. If ``None``,+the User-Agent header you are sending with the request or the+:setting:`USER_AGENT` setting (in that order) will be used for determining+the user agent to use in the robots.txt file.+.. setting:: SCHEDULERSCHEDULER@@ -1409,7 +1421,9 @@ USER_AGENTDefault: ``"Scrapy/VERSION (+https://scrapy.org)"``-The default User-Agent to use when crawling, unless overridden.+The default User-Agent to use when crawling, unless overridden. This user agent is+also used in robots.txt if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and+there is no overridding User-Agent header specified for the request.Settings documented elsewhere:diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex c5a60d355..6a5dfb79c 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -26,6 +26,7 @@ class RobotsTxtMiddleware(object):if not crawler.settings.getbool('ROBOTSTXT_OBEY'):raise NotConfiguredself._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')+ self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)self.crawler = crawlerself._parsers = {}self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))@@ -47,7 +48,10 @@ class RobotsTxtMiddleware(object):def process_request_2(self, rp, request, spider):if rp is None:return- useragent = request.headers.get(b'User-Agent', self._default_useragent)++ useragent = self._robotstxt_useragent+ if not useragent:+ useragent = request.headers.get(b'User-Agent', self._default_useragent)if not rp.allowed(request.url, useragent):logger.debug("Forbidden by robots.txt: %(request)s",{'request': request}, extra={'spider': spider})diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 81fee543f..efb257e2b 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -246,6 +246,7 @@ RETRY_PRIORITY_ADJUST = -1ROBOTSTXT_OBEY = FalseROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'+ROBOTSTXT_USER_AGENT = NoneSCHEDULER = 'scrapy.core.scheduler.Scheduler'SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 79f172848..fbc46cba4 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -164,6 +164,15 @@ Disallow: /some/randome/page.htmld.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))return d+ def test_robotstxt_user_agent_setting(self):+ crawler = self._get_successful_crawler()+ crawler.settings.set('ROBOTSTXT_USER_AGENT', 'Examplebot')+ crawler.settings.set('USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64)')+ middleware = RobotsTxtMiddleware(crawler)+ rp = mock.MagicMock(return_value=True)+ middleware.process_request_2(rp, Request('http://site.local/allowed'), None)+ rp.allowed.assert_called_once_with('http://site.local/allowed', 'Examplebot')+def assertNotIgnored(self, request, middleware):spider = None # not actually useddfd = maybeDeferred(middleware.process_request, request, spider)