Scrapy, a fast high-level web crawling & scraping framework for Python.
Adds integration with Protego robots.txt parser (#3935)
Anubhav Patel3a7b949d6d80
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 616b56101..6aa714fb2 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -996,6 +996,7 @@ RobotsTxtMiddleware* :ref:`RobotFileParser <python-robotfileparser>` (default)* :ref:`Reppy <reppy-parser>`* :ref:`Robotexclusionrulesparser <rerp-parser>`+ * :ref:`Protego <protego-parser>`You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.@@ -1013,7 +1014,7 @@ RobotFileParser~~~~~~~~~~~~~~~`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is-Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's+Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lackssupport for wildcard matching. Scrapy uses this parser by default.@@ -1059,6 +1060,23 @@ In order to use this parser:* Set :setting:`ROBOTSTXT_PARSER` setting to``scrapy.robotstxt.ReppyRobotParser``+.. _protego-parser:++Protego parser+~~~~~~~~~~~~~~++`Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.+The parser is fully compliant with `Google's Robots.txt Specification+<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard+matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.++In order to use this parser:++* Install `Protego <https://github.com/scrapy/protego>`_ by running ``pip install protego``++* Set :setting:`ROBOTSTXT_PARSER` setting to+ ``scrapy.robotstxt.ProtegoRobotParser``+.. _support-for-new-robots-parser:Implementing support for a new parserdiff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 4bfb275fd..189f165d1 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -7,6 +7,21 @@ from scrapy.utils.python import to_native_str, to_unicodelogger = logging.getLogger(__name__)+def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):+ try:+ if to_native_str_type:+ robotstxt_body = to_native_str(robotstxt_body)+ else:+ robotstxt_body = robotstxt_body.decode('utf-8')+ except UnicodeDecodeError:+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.+ # Switch to 'allow all' state.+ logger.warning("Failure while parsing robots.txt. "+ "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",+ exc_info=sys.exc_info(),+ extra={'spider': spider})+ robotstxt_body = ''+ return robotstxt_bodyclass RobotParser(with_metaclass(ABCMeta)):@classmethod@@ -40,17 +55,7 @@ class PythonRobotParser(RobotParser):def __init__(self, robotstxt_body, spider):from six.moves.urllib_robotparser import RobotFileParserself.spider = spider- try:- robotstxt_body = to_native_str(robotstxt_body)- except UnicodeDecodeError:- # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.- # Switch to 'allow all' state.- logger.warning("Failure while parsing robots.txt using %(parser)s."- " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",- {'parser': "RobotFileParser"},- exc_info=sys.exc_info(),- extra={'spider': self.spider})- robotstxt_body = ''+ robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)self.rp = RobotFileParser()self.rp.parse(robotstxt_body.splitlines())@@ -87,17 +92,7 @@ class RerpRobotParser(RobotParser):from robotexclusionrulesparser import RobotExclusionRulesParserself.spider = spiderself.rp = RobotExclusionRulesParser()- try:- robotstxt_body = robotstxt_body.decode('utf-8')- except UnicodeDecodeError:- # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.- # Switch to 'allow all' state.- logger.warning("Failure while parsing robots.txt using %(parser)s."- " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",- {'parser': "RobotExclusionRulesParser"},- exc_info=sys.exc_info(),- extra={'spider': self.spider})- robotstxt_body = ''+ robotstxt_body = decode_robotstxt(robotstxt_body, spider)self.rp.parse(robotstxt_body)@classmethod@@ -110,3 +105,22 @@ class RerpRobotParser(RobotParser):user_agent = to_unicode(user_agent)url = to_unicode(url)return self.rp.is_allowed(user_agent, url)+++class ProtegoRobotParser(RobotParser):+ def __init__(self, robotstxt_body, spider):+ from protego import Protego+ self.spider = spider+ robotstxt_body = decode_robotstxt(robotstxt_body, spider)+ self.rp = Protego.parse(robotstxt_body)++ @classmethod+ def from_crawler(cls, crawler, robotstxt_body):+ spider = None if not crawler else crawler.spider+ o = cls(robotstxt_body, spider)+ return o++ def allowed(self, url, user_agent):+ user_agent = to_unicode(user_agent)+ url = to_unicode(url)+ return self.rp.can_fetch(url, user_agent)diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 2819786b5..9aaab560a 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -20,6 +20,13 @@ def rerp_available():return Falsereturn True+def protego_available():+ # check if protego parser is installed+ try:+ from protego import Protego+ except ImportError:+ return False+ return Trueclass BaseRobotParserTest:def _setUp(self, parser_cls):@@ -127,7 +134,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)def test_order_based_precedence(self):- raise unittest.SkipTest("Rerp does not support order based directives precedence.")+ raise unittest.SkipTest("Reppy does not support order based directives precedence.")class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):@@ -140,3 +147,15 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):def test_length_based_precedence(self):raise unittest.SkipTest("Rerp does not support length based directives precedence.")+++class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):+ if not protego_available():+ skip = "Protego parser is not installed"++ def setUp(self):+ from scrapy.robotstxt import ProtegoRobotParser+ super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)++ def test_order_based_precedence(self):+ raise unittest.SkipTest("Protego does not support order based directives precedence.")diff --git a/tox.ini b/tox.iniindex c3502c2ca..cc845faf1 100644--- a/tox.ini+++ b/tox.ini@@ -125,6 +125,7 @@ deps ={[testenv:py35]deps}reppyrobotexclusionrulesparser+ protego[testenv:py27-extra-deps]basepython = python2.7@@ -132,3 +133,4 @@ deps ={[testenv]deps}reppyrobotexclusionrulesparser+ protego\ No newline at end of file