Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Adds integration with Protego robots.txt parser (#3935)

Anubhav Patel3a7b949d6d80
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 616b56101..6aa714fb2 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -996,6 +996,7 @@ RobotsTxtMiddleware
* :ref:`RobotFileParser <python-robotfileparser>` (default)
* :ref:`Reppy <reppy-parser>`
* :ref:`Robotexclusionrulesparser <rerp-parser>`
+ * :ref:`Protego <protego-parser>`
You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1013,7 +1014,7 @@ RobotFileParser
~~~~~~~~~~~~~~~
`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is
-Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's
+Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's
1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
support for wildcard matching. Scrapy uses this parser by default.
@@ -1059,6 +1060,23 @@ In order to use this parser:
* Set :setting:`ROBOTSTXT_PARSER` setting to
``scrapy.robotstxt.ReppyRobotParser``
+.. _protego-parser:
+
+Protego parser
+~~~~~~~~~~~~~~
+
+`Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.
+The parser is fully compliant with `Google's Robots.txt Specification
+<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard
+matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.
+
+In order to use this parser:
+
+* Install `Protego <https://github.com/scrapy/protego>`_ by running ``pip install protego``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+ ``scrapy.robotstxt.ProtegoRobotParser``
+
.. _support-for-new-robots-parser:
Implementing support for a new parser
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 4bfb275fd..189f165d1 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -7,6 +7,21 @@ from scrapy.utils.python import to_native_str, to_unicode
logger = logging.getLogger(__name__)
+def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
+ try:
+ if to_native_str_type:
+ robotstxt_body = to_native_str(robotstxt_body)
+ else:
+ robotstxt_body = robotstxt_body.decode('utf-8')
+ except UnicodeDecodeError:
+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+ # Switch to 'allow all' state.
+ logger.warning("Failure while parsing robots.txt. "
+ "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+ exc_info=sys.exc_info(),
+ extra={'spider': spider})
+ robotstxt_body = ''
+ return robotstxt_body
class RobotParser(with_metaclass(ABCMeta)):
@classmethod
@@ -40,17 +55,7 @@ class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
from six.moves.urllib_robotparser import RobotFileParser
self.spider = spider
- try:
- robotstxt_body = to_native_str(robotstxt_body)
- except UnicodeDecodeError:
- # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
- # Switch to 'allow all' state.
- logger.warning("Failure while parsing robots.txt using %(parser)s."
- " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
- {'parser': "RobotFileParser"},
- exc_info=sys.exc_info(),
- extra={'spider': self.spider})
- robotstxt_body = ''
+ robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp = RobotFileParser()
self.rp.parse(robotstxt_body.splitlines())
@@ -87,17 +92,7 @@ class RerpRobotParser(RobotParser):
from robotexclusionrulesparser import RobotExclusionRulesParser
self.spider = spider
self.rp = RobotExclusionRulesParser()
- try:
- robotstxt_body = robotstxt_body.decode('utf-8')
- except UnicodeDecodeError:
- # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
- # Switch to 'allow all' state.
- logger.warning("Failure while parsing robots.txt using %(parser)s."
- " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
- {'parser': "RobotExclusionRulesParser"},
- exc_info=sys.exc_info(),
- extra={'spider': self.spider})
- robotstxt_body = ''
+ robotstxt_body = decode_robotstxt(robotstxt_body, spider)
self.rp.parse(robotstxt_body)
@classmethod
@@ -110,3 +105,22 @@ class RerpRobotParser(RobotParser):
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.is_allowed(user_agent, url)
+
+
+class ProtegoRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from protego import Protego
+ self.spider = spider
+ robotstxt_body = decode_robotstxt(robotstxt_body, spider)
+ self.rp = Protego.parse(robotstxt_body)
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ user_agent = to_unicode(user_agent)
+ url = to_unicode(url)
+ return self.rp.can_fetch(url, user_agent)
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 2819786b5..9aaab560a 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -20,6 +20,13 @@ def rerp_available():
return False
return True
+def protego_available():
+ # check if protego parser is installed
+ try:
+ from protego import Protego
+ except ImportError:
+ return False
+ return True
class BaseRobotParserTest:
def _setUp(self, parser_cls):
@@ -127,7 +134,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
def test_order_based_precedence(self):
- raise unittest.SkipTest("Rerp does not support order based directives precedence.")
+ raise unittest.SkipTest("Reppy does not support order based directives precedence.")
class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
@@ -140,3 +147,15 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def test_length_based_precedence(self):
raise unittest.SkipTest("Rerp does not support length based directives precedence.")
+
+
+class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+ if not protego_available():
+ skip = "Protego parser is not installed"
+
+ def setUp(self):
+ from scrapy.robotstxt import ProtegoRobotParser
+ super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
+
+ def test_order_based_precedence(self):
+ raise unittest.SkipTest("Protego does not support order based directives precedence.")
diff --git a/tox.ini b/tox.ini
index c3502c2ca..cc845faf1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -125,6 +125,7 @@ deps =
{[testenv:py35]deps}
reppy
robotexclusionrulesparser
+ protego
[testenv:py27-extra-deps]
basepython = python2.7
@@ -132,3 +133,4 @@ deps =
{[testenv]deps}
reppy
robotexclusionrulesparser
+ protego
\ No newline at end of file