Scrapy, a fast high-level web crawling & scraping framework for Python.
[MRG+1] [GSoC 2019] Interface for robots.txt parsers (#3796)
Anubhav Patel8e813953bd4b
diff --git a/.travis.yml b/.travis.ymlindex 3116d9b48..138e81c64 100644--- a/.travis.yml+++ b/.travis.yml@@ -27,6 +27,12 @@ matrix:sudo: true- python: 3.6env: TOXENV=docs+ - python: 3.7+ env: TOXENV=py37-extra-deps+ dist: xenial+ sudo: true+ - python: 2.7+ env: TOXENV=py27-extra-depsinstall:- |if [ "$TOXENV" = "pypy" ]; thendiff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex a3780a177..616b56101 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -989,6 +989,17 @@ RobotsTxtMiddlewareTo make sure Scrapy respects robots.txt make sure the middleware is enabledand the :setting:`ROBOTSTXT_OBEY` setting is enabled.+ This middleware has to be combined with a robots.txt_ parser.++ Scrapy ships with support for the following robots.txt_ parsers:++ * :ref:`RobotFileParser <python-robotfileparser>` (default)+ * :ref:`Reppy <reppy-parser>`+ * :ref:`Robotexclusionrulesparser <rerp-parser>`++ You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`+ setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.+.. reqmeta:: dont_obey_robotstxtIf :attr:`Request.meta <scrapy.http.Request.meta>` has@@ -996,6 +1007,74 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` hasthe request will be ignored by this middleware even if:setting:`ROBOTSTXT_OBEY` is enabled.+.. _python-robotfileparser:++RobotFileParser+~~~~~~~~~~~~~~~++`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is+Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's+1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks+support for wildcard matching. Scrapy uses this parser by default.++In order to use this parser, set:++* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``++.. _rerp-parser:++Robotexclusionrulesparser+~~~~~~~~~~~~~~~~~~~~~~~~~++`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ is fully compliant+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,+with support for wildcard matching.++In order to use this parser:++* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running+ ``pip install robotexclusionrulesparser``++* Set :setting:`ROBOTSTXT_PARSER` setting to+ ``scrapy.robotstxt.RerpRobotParser``++.. _reppy-parser:++Reppy parser+~~~~~~~~~~~~++`Reppy <https://github.com/seomoz/reppy/>`_ is a Python wrapper around `Robots Exclusion+Protocol Parser for C++ <https://github.com/seomoz/rep-cpp>`_. The parser is fully compliant+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,+with support for wildcard matching. Unlike+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ and+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_, it uses the length based+rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific+rule based on the length of the path trumps the less specific (shorter) rule.++In order to use this parser:++* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``++* Set :setting:`ROBOTSTXT_PARSER` setting to+ ``scrapy.robotstxt.ReppyRobotParser``++.. _support-for-new-robots-parser:++Implementing support for a new parser+-------------------------------------++You can implement support for a new robots.txt_ parser by subclassing+the abstract base class :class:`~scrapy.robotstxt.RobotParser` and+implementing the methods described below.++.. module:: scrapy.robotstxt+ :synopsis: robots.txt parser interface and implementations++.. autoclass:: RobotParser+ :members:++.. _robots.txt: http://www.robotstxt.org/DownloaderStats---------------diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 85ae2a305..12606fe47 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1141,6 +1141,16 @@ If enabled, Scrapy will respect robots.txt policies. For more information seethis option is enabled by default in settings.py file generatedby ``scrapy startproject`` command.+.. setting:: ROBOTSTXT_PARSER++ROBOTSTXT_PARSER+----------------++Default: ``'scrapy.robotstxt.PythonRobotParser'``++The parser backend to use for parsing ``robots.txt`` files. For more information see+:ref:`topics-dlmw-robots`.+.. setting:: SCHEDULERSCHEDULERdiff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 200245210..c5a60d355 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -5,8 +5,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting."""import logging--from six.moves.urllib import robotparser+import sys+import refrom twisted.internet.defer import Deferred, maybeDeferredfrom scrapy.exceptions import NotConfigured, IgnoreRequest@@ -14,6 +14,7 @@ from scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.python import to_native_str+from scrapy.utils.misc import load_objectlogger = logging.getLogger(__name__)@@ -24,10 +25,13 @@ class RobotsTxtMiddleware(object):def __init__(self, crawler):if not crawler.settings.getbool('ROBOTSTXT_OBEY'):raise NotConfigured-+ self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')self.crawler = crawler- self._useragent = crawler.settings.get('USER_AGENT')self._parsers = {}+ self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))++ # check if parser dependencies are met, this should throw an error otherwise.+ self._parserimpl.from_crawler(self.crawler, b'')@classmethoddef from_crawler(cls, crawler):@@ -43,7 +47,8 @@ class RobotsTxtMiddleware(object):def process_request_2(self, rp, request, spider):if rp is None:return- if not rp.can_fetch(to_native_str(self._useragent), request.url):+ useragent = request.headers.get(b'User-Agent', self._default_useragent)+ if not rp.allowed(request.url, useragent):logger.debug("Forbidden by robots.txt: %(request)s",{'request': request}, extra={'spider': spider})self.crawler.stats.inc_value('robotstxt/forbidden')@@ -62,13 +67,14 @@ class RobotsTxtMiddleware(object):meta={'dont_obey_robotstxt': True})dfd = self.crawler.engine.download(robotsreq, spider)- dfd.addCallback(self._parse_robots, netloc)+ dfd.addCallback(self._parse_robots, netloc, spider)dfd.addErrback(self._logerror, robotsreq, spider)dfd.addErrback(self._robots_error, netloc)self.crawler.stats.inc_value('robotstxt/request_count')if isinstance(self._parsers[netloc], Deferred):d = Deferred()+def cb(result):d.callback(result)return result@@ -85,27 +91,10 @@ class RobotsTxtMiddleware(object):extra={'spider': spider})return failure- def _parse_robots(self, response, netloc):+ def _parse_robots(self, response, netloc, spider):self.crawler.stats.inc_value('robotstxt/response_count')- self.crawler.stats.inc_value(- 'robotstxt/response_status_count/{}'.format(response.status))- rp = robotparser.RobotFileParser(response.url)- body = ''- if hasattr(response, 'text'):- body = response.text- else: # last effort try- try:- body = response.body.decode('utf-8')- except UnicodeDecodeError:- # If we found garbage, disregard it:,- # but keep the lookup cached (in self._parsers)- # Running rp.parse() will set rp state from- # 'disallow all' to 'allow any'.- self.crawler.stats.inc_value('robotstxt/unicode_error_count')- # stdlib's robotparser expects native 'str' ;- # with unicode input, non-ASCII encoded bytes decoding fails in Python2- rp.parse(to_native_str(body).splitlines())-+ self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))+ rp = self._parserimpl.from_crawler(self.crawler, response.body)rp_dfd = self._parsers[netloc]self._parsers[netloc] = rprp_dfd.callback(rp)diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pynew file mode 100644index 000000000..4bfb275fd--- /dev/null+++ b/scrapy/robotstxt.py@@ -0,0 +1,112 @@+import sys+import logging+from abc import ABCMeta, abstractmethod+from six import with_metaclass++from scrapy.utils.python import to_native_str, to_unicode++logger = logging.getLogger(__name__)+++class RobotParser(with_metaclass(ABCMeta)):+ @classmethod+ @abstractmethod+ def from_crawler(cls, crawler, robotstxt_body):+ """Parse the content of a robots.txt_ file as bytes. This must be a class method.+ It must return a new instance of the parser backend.++ :param crawler: crawler which made the request+ :type crawler: :class:`~scrapy.crawler.Crawler` instance++ :param robotstxt_body: content of a robots.txt_ file.+ :type robotstxt_body: bytes+ """+ pass++ @abstractmethod+ def allowed(self, url, user_agent):+ """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.++ :param url: Absolute URL+ :type url: string++ :param user_agent: User agent+ :type user_agent: string+ """+ pass+++class PythonRobotParser(RobotParser):+ def __init__(self, robotstxt_body, spider):+ from six.moves.urllib_robotparser import RobotFileParser+ self.spider = spider+ try:+ robotstxt_body = to_native_str(robotstxt_body)+ except UnicodeDecodeError:+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.+ # Switch to 'allow all' state.+ logger.warning("Failure while parsing robots.txt using %(parser)s."+ " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",+ {'parser': "RobotFileParser"},+ exc_info=sys.exc_info(),+ extra={'spider': self.spider})+ robotstxt_body = ''+ self.rp = RobotFileParser()+ self.rp.parse(robotstxt_body.splitlines())++ @classmethod+ def from_crawler(cls, crawler, robotstxt_body):+ spider = None if not crawler else crawler.spider+ o = cls(robotstxt_body, spider)+ return o++ def allowed(self, url, user_agent):+ user_agent = to_native_str(user_agent)+ url = to_native_str(url)+ return self.rp.can_fetch(user_agent, url)+++class ReppyRobotParser(RobotParser):+ def __init__(self, robotstxt_body, spider):+ from reppy.robots import Robots+ self.spider = spider+ self.rp = Robots.parse('', robotstxt_body)++ @classmethod+ def from_crawler(cls, crawler, robotstxt_body):+ spider = None if not crawler else crawler.spider+ o = cls(robotstxt_body, spider)+ return o++ def allowed(self, url, user_agent):+ return self.rp.allowed(url, user_agent)+++class RerpRobotParser(RobotParser):+ def __init__(self, robotstxt_body, spider):+ from robotexclusionrulesparser import RobotExclusionRulesParser+ self.spider = spider+ self.rp = RobotExclusionRulesParser()+ try:+ robotstxt_body = robotstxt_body.decode('utf-8')+ except UnicodeDecodeError:+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.+ # Switch to 'allow all' state.+ logger.warning("Failure while parsing robots.txt using %(parser)s."+ " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",+ {'parser': "RobotExclusionRulesParser"},+ exc_info=sys.exc_info(),+ extra={'spider': self.spider})+ robotstxt_body = ''+ self.rp.parse(robotstxt_body)++ @classmethod+ def from_crawler(cls, crawler, robotstxt_body):+ spider = None if not crawler else crawler.spider+ o = cls(robotstxt_body, spider)+ return o++ def allowed(self, url, user_agent):+ user_agent = to_unicode(user_agent)+ url = to_unicode(url)+ return self.rp.is_allowed(user_agent, url)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 086adf48e..81fee543f 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -245,6 +245,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]RETRY_PRIORITY_ADJUST = -1ROBOTSTXT_OBEY = False+ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'SCHEDULER = 'scrapy.core.scheduler.Scheduler'SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 2b3548bdd..79f172848 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -10,6 +10,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponsefrom scrapy.settings import Settingsfrom tests import mock+from tests.test_robotstxt_interface import rerp_available, reppy_availableclass RobotsTxtMiddlewareTest(unittest.TestCase):@@ -41,6 +42,7 @@ User-Agent: UnicödeBötDisallow: /some/randome/page.html""".encode('utf-8')response = TextResponse('http://site.local/robots.txt', body=ROBOTS)+def return_response(request, spider):deferred = Deferred()reactor.callFromThread(deferred.callback, response)@@ -77,6 +79,7 @@ Disallow: /some/randome/page.htmlcrawler = self.crawlercrawler.settings.set('ROBOTSTXT_OBEY', True)response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')+def return_response(request, spider):deferred = Deferred()reactor.callFromThread(deferred.callback, response)@@ -99,6 +102,7 @@ Disallow: /some/randome/page.htmlcrawler = self.crawlercrawler.settings.set('ROBOTSTXT_OBEY', True)response = Response('http://site.local/robots.txt')+def return_response(request, spider):deferred = Deferred()reactor.callFromThread(deferred.callback, response)@@ -118,6 +122,7 @@ Disallow: /some/randome/page.htmldef test_robotstxt_error(self):self.crawler.settings.set('ROBOTSTXT_OBEY', True)err = error.DNSLookupError('Robotstxt address not found')+def return_failure(request, spider):deferred = Deferred()reactor.callFromThread(deferred.errback, failure.Failure(err))@@ -133,6 +138,7 @@ Disallow: /some/randome/page.htmldef test_robotstxt_immediate_error(self):self.crawler.settings.set('ROBOTSTXT_OBEY', True)err = error.DNSLookupError('Robotstxt address not found')+def immediate_failure(request, spider):deferred = Deferred()deferred.errback(failure.Failure(err))@@ -144,6 +150,7 @@ Disallow: /some/randome/page.htmldef test_ignore_robotstxt_request(self):self.crawler.settings.set('ROBOTSTXT_OBEY', True)+def ignore_request(request, spider):deferred = Deferred()reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))@@ -167,3 +174,21 @@ Disallow: /some/randome/page.htmlspider = None # not actually usedreturn self.assertFailure(maybeDeferred(middleware.process_request, request, spider),IgnoreRequest)+++class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):+ if not rerp_available():+ skip = "Rerp parser is not installed"++ def setUp(self):+ super(RobotsTxtMiddlewareWithRerpTest, self).setUp()+ self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')+++class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):+ if not reppy_available():+ skip = "Reppy parser is not installed"++ def setUp(self):+ super(RobotsTxtMiddlewareWithReppyTest, self).setUp()+ self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pynew file mode 100644index 000000000..2819786b5--- /dev/null+++ b/tests/test_robotstxt_interface.py@@ -0,0 +1,142 @@+# coding=utf-8+from twisted.trial import unittest+from scrapy.utils.python import to_native_str+++def reppy_available():+ # check if reppy parser is installed+ try:+ from reppy.robots import Robots+ except ImportError:+ return False+ return True+++def rerp_available():+ # check if robotexclusionrulesparser is installed+ try:+ from robotexclusionrulesparser import RobotExclusionRulesParser+ except ImportError:+ return False+ return True+++class BaseRobotParserTest:+ def _setUp(self, parser_cls):+ self.parser_cls = parser_cls++ def test_allowed(self):+ robotstxt_robotstxt_body = ("User-agent: * \n"+ "Disallow: /disallowed \n"+ "Allow: /allowed \n"+ "Crawl-delay: 10".encode('utf-8'))+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)+ self.assertTrue(rp.allowed("https://www.site.local/allowed", "*"))+ self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))++ def test_allowed_wildcards(self):+ robotstxt_robotstxt_body = """User-agent: first+ Disallow: /disallowed/*/end$++ User-agent: second+ Allow: /*allowed+ Disallow: /+ """.encode('utf-8')+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)++ self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first"))+ self.assertFalse(rp.allowed("https://www.site.local/disallowed/xyz/end", "first"))+ self.assertFalse(rp.allowed("https://www.site.local/disallowed/abc/end", "first"))+ self.assertTrue(rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first"))++ self.assertTrue(rp.allowed("https://www.site.local/allowed", "second"))+ self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second"))+ self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))++ def test_length_based_precedence(self):+ robotstxt_robotstxt_body = ("User-agent: * \n"+ "Disallow: / \n"+ "Allow: /page".encode('utf-8'))+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)+ self.assertTrue(rp.allowed("https://www.site.local/page", "*"))++ def test_order_based_precedence(self):+ robotstxt_robotstxt_body = ("User-agent: * \n"+ "Disallow: / \n"+ "Allow: /page".encode('utf-8'))+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)+ self.assertFalse(rp.allowed("https://www.site.local/page", "*"))++ def test_empty_response(self):+ """empty response should equal 'allow all'"""+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b'')+ self.assertTrue(rp.allowed("https://site.local/", "*"))+ self.assertTrue(rp.allowed("https://site.local/", "chrome"))+ self.assertTrue(rp.allowed("https://site.local/index.html", "*"))+ self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))++ def test_garbage_response(self):+ """garbage response should be discarded, equal 'allow all'"""+ robotstxt_robotstxt_body = b'GIF89a\xd3\x00\xfe\x00\xa2'+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)+ self.assertTrue(rp.allowed("https://site.local/", "*"))+ self.assertTrue(rp.allowed("https://site.local/", "chrome"))+ self.assertTrue(rp.allowed("https://site.local/index.html", "*"))+ self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))++ def test_unicode_url_and_useragent(self):+ robotstxt_robotstxt_body = u"""+ User-Agent: *+ Disallow: /admin/+ Disallow: /static/+ # taken from https://en.wikipedia.org/robots.txt+ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:+ Disallow: /wiki/Käyttäjä:++ User-Agent: UnicödeBöt+ Disallow: /some/randome/page.html""".encode('utf-8')+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)+ self.assertTrue(rp.allowed("https://site.local/", "*"))+ self.assertFalse(rp.allowed("https://site.local/admin/", "*"))+ self.assertFalse(rp.allowed("https://site.local/static/", "*"))+ self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))+ self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))+ self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))+ self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))+ self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))+++class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):+ def setUp(self):+ from scrapy.robotstxt import PythonRobotParser+ super(PythonRobotParserTest, self)._setUp(PythonRobotParser)++ def test_length_based_precedence(self):+ raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")++ def test_allowed_wildcards(self):+ raise unittest.SkipTest("RobotFileParser does not support wildcards.")+++class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):+ if not reppy_available():+ skip = "Reppy parser is not installed"++ def setUp(self):+ from scrapy.robotstxt import ReppyRobotParser+ super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)++ def test_order_based_precedence(self):+ raise unittest.SkipTest("Rerp does not support order based directives precedence.")+++class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):+ if not rerp_available():+ skip = "Rerp parser is not installed"++ def setUp(self):+ from scrapy.robotstxt import RerpRobotParser+ super(RerpRobotParserTest, self)._setUp(RerpRobotParser)++ def test_length_based_precedence(self):+ raise unittest.SkipTest("Rerp does not support length based directives precedence.")diff --git a/tox.ini b/tox.iniindex 157a8b3ed..c918731f4 100644--- a/tox.ini+++ b/tox.ini@@ -116,3 +116,17 @@ changedir = {[docs]changedir}deps = {[docs]deps}commands =sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck++[testenv:py37-extra-deps]+basepython = python3.7+deps =+ {[testenv:py34]deps}+ reppy+ robotexclusionrulesparser++[testenv:py27-extra-deps]+basepython = python2.7+deps =+ {[testenv]deps}+ reppy+ robotexclusionrulesparser