Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[MRG+1] [GSoC 2019] Interface for robots.txt parsers (#3796)

Anubhav Patel8e813953bd4b
diff --git a/.travis.yml b/.travis.yml
index 3116d9b48..138e81c64 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,12 @@ matrix:
sudo: true
- python: 3.6
env: TOXENV=docs
+ - python: 3.7
+ env: TOXENV=py37-extra-deps
+ dist: xenial
+ sudo: true
+ - python: 2.7
+ env: TOXENV=py27-extra-deps
install:
- |
if [ "$TOXENV" = "pypy" ]; then
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a3780a177..616b56101 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -989,6 +989,17 @@ RobotsTxtMiddleware
To make sure Scrapy respects robots.txt make sure the middleware is enabled
and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+ This middleware has to be combined with a robots.txt_ parser.
+
+ Scrapy ships with support for the following robots.txt_ parsers:
+
+ * :ref:`RobotFileParser <python-robotfileparser>` (default)
+ * :ref:`Reppy <reppy-parser>`
+ * :ref:`Robotexclusionrulesparser <rerp-parser>`
+
+ You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+ setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
+
.. reqmeta:: dont_obey_robotstxt
If :attr:`Request.meta <scrapy.http.Request.meta>` has
@@ -996,6 +1007,74 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
the request will be ignored by this middleware even if
:setting:`ROBOTSTXT_OBEY` is enabled.
+.. _python-robotfileparser:
+
+RobotFileParser
+~~~~~~~~~~~~~~~
+
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is
+Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's
+1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
+support for wildcard matching. Scrapy uses this parser by default.
+
+In order to use this parser, set:
+
+* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
+
+.. _rerp-parser:
+
+Robotexclusionrulesparser
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ is fully compliant
+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
+with support for wildcard matching.
+
+In order to use this parser:
+
+* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
+ ``pip install robotexclusionrulesparser``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+ ``scrapy.robotstxt.RerpRobotParser``
+
+.. _reppy-parser:
+
+Reppy parser
+~~~~~~~~~~~~
+
+`Reppy <https://github.com/seomoz/reppy/>`_ is a Python wrapper around `Robots Exclusion
+Protocol Parser for C++ <https://github.com/seomoz/rep-cpp>`_. The parser is fully compliant
+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
+with support for wildcard matching. Unlike
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ and
+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_, it uses the length based
+rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific
+rule based on the length of the path trumps the less specific (shorter) rule.
+
+In order to use this parser:
+
+* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+ ``scrapy.robotstxt.ReppyRobotParser``
+
+.. _support-for-new-robots-parser:
+
+Implementing support for a new parser
+-------------------------------------
+
+You can implement support for a new robots.txt_ parser by subclassing
+the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
+implementing the methods described below.
+
+.. module:: scrapy.robotstxt
+ :synopsis: robots.txt parser interface and implementations
+
+.. autoclass:: RobotParser
+ :members:
+
+.. _robots.txt: http://www.robotstxt.org/
DownloaderStats
---------------
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 85ae2a305..12606fe47 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1141,6 +1141,16 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
this option is enabled by default in settings.py file generated
by ``scrapy startproject`` command.
+.. setting:: ROBOTSTXT_PARSER
+
+ROBOTSTXT_PARSER
+----------------
+
+Default: ``'scrapy.robotstxt.PythonRobotParser'``
+
+The parser backend to use for parsing ``robots.txt`` files. For more information see
+:ref:`topics-dlmw-robots`.
+
.. setting:: SCHEDULER
SCHEDULER
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 200245210..c5a60d355 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -5,8 +5,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
"""
import logging
-
-from six.moves.urllib import robotparser
+import sys
+import re
from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import NotConfigured, IgnoreRequest
@@ -14,6 +14,7 @@ from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import to_native_str
+from scrapy.utils.misc import load_object
logger = logging.getLogger(__name__)
@@ -24,10 +25,13 @@ class RobotsTxtMiddleware(object):
def __init__(self, crawler):
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
raise NotConfigured
-
+ self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
self.crawler = crawler
- self._useragent = crawler.settings.get('USER_AGENT')
self._parsers = {}
+ self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
+
+ # check if parser dependencies are met, this should throw an error otherwise.
+ self._parserimpl.from_crawler(self.crawler, b'')
@classmethod
def from_crawler(cls, crawler):
@@ -43,7 +47,8 @@ class RobotsTxtMiddleware(object):
def process_request_2(self, rp, request, spider):
if rp is None:
return
- if not rp.can_fetch(to_native_str(self._useragent), request.url):
+ useragent = request.headers.get(b'User-Agent', self._default_useragent)
+ if not rp.allowed(request.url, useragent):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
self.crawler.stats.inc_value('robotstxt/forbidden')
@@ -62,13 +67,14 @@ class RobotsTxtMiddleware(object):
meta={'dont_obey_robotstxt': True}
)
dfd = self.crawler.engine.download(robotsreq, spider)
- dfd.addCallback(self._parse_robots, netloc)
+ dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value('robotstxt/request_count')
if isinstance(self._parsers[netloc], Deferred):
d = Deferred()
+
def cb(result):
d.callback(result)
return result
@@ -85,27 +91,10 @@ class RobotsTxtMiddleware(object):
extra={'spider': spider})
return failure
- def _parse_robots(self, response, netloc):
+ def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
- self.crawler.stats.inc_value(
- 'robotstxt/response_status_count/{}'.format(response.status))
- rp = robotparser.RobotFileParser(response.url)
- body = ''
- if hasattr(response, 'text'):
- body = response.text
- else: # last effort try
- try:
- body = response.body.decode('utf-8')
- except UnicodeDecodeError:
- # If we found garbage, disregard it:,
- # but keep the lookup cached (in self._parsers)
- # Running rp.parse() will set rp state from
- # 'disallow all' to 'allow any'.
- self.crawler.stats.inc_value('robotstxt/unicode_error_count')
- # stdlib's robotparser expects native 'str' ;
- # with unicode input, non-ASCII encoded bytes decoding fails in Python2
- rp.parse(to_native_str(body).splitlines())
-
+ self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
+ rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
rp_dfd.callback(rp)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
new file mode 100644
index 000000000..4bfb275fd
--- /dev/null
+++ b/scrapy/robotstxt.py
@@ -0,0 +1,112 @@
+import sys
+import logging
+from abc import ABCMeta, abstractmethod
+from six import with_metaclass
+
+from scrapy.utils.python import to_native_str, to_unicode
+
+logger = logging.getLogger(__name__)
+
+
+class RobotParser(with_metaclass(ABCMeta)):
+ @classmethod
+ @abstractmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ """Parse the content of a robots.txt_ file as bytes. This must be a class method.
+ It must return a new instance of the parser backend.
+
+ :param crawler: crawler which made the request
+ :type crawler: :class:`~scrapy.crawler.Crawler` instance
+
+ :param robotstxt_body: content of a robots.txt_ file.
+ :type robotstxt_body: bytes
+ """
+ pass
+
+ @abstractmethod
+ def allowed(self, url, user_agent):
+ """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
+
+ :param url: Absolute URL
+ :type url: string
+
+ :param user_agent: User agent
+ :type user_agent: string
+ """
+ pass
+
+
+class PythonRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from six.moves.urllib_robotparser import RobotFileParser
+ self.spider = spider
+ try:
+ robotstxt_body = to_native_str(robotstxt_body)
+ except UnicodeDecodeError:
+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+ # Switch to 'allow all' state.
+ logger.warning("Failure while parsing robots.txt using %(parser)s."
+ " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+ {'parser': "RobotFileParser"},
+ exc_info=sys.exc_info(),
+ extra={'spider': self.spider})
+ robotstxt_body = ''
+ self.rp = RobotFileParser()
+ self.rp.parse(robotstxt_body.splitlines())
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ user_agent = to_native_str(user_agent)
+ url = to_native_str(url)
+ return self.rp.can_fetch(user_agent, url)
+
+
+class ReppyRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from reppy.robots import Robots
+ self.spider = spider
+ self.rp = Robots.parse('', robotstxt_body)
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ return self.rp.allowed(url, user_agent)
+
+
+class RerpRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from robotexclusionrulesparser import RobotExclusionRulesParser
+ self.spider = spider
+ self.rp = RobotExclusionRulesParser()
+ try:
+ robotstxt_body = robotstxt_body.decode('utf-8')
+ except UnicodeDecodeError:
+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+ # Switch to 'allow all' state.
+ logger.warning("Failure while parsing robots.txt using %(parser)s."
+ " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+ {'parser': "RobotExclusionRulesParser"},
+ exc_info=sys.exc_info(),
+ extra={'spider': self.spider})
+ robotstxt_body = ''
+ self.rp.parse(robotstxt_body)
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ user_agent = to_unicode(user_agent)
+ url = to_unicode(url)
+ return self.rp.is_allowed(user_agent, url)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 086adf48e..81fee543f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -245,6 +245,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
RETRY_PRIORITY_ADJUST = -1
ROBOTSTXT_OBEY = False
+ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 2b3548bdd..79f172848 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -10,6 +10,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
from scrapy.settings import Settings
from tests import mock
+from tests.test_robotstxt_interface import rerp_available, reppy_available
class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -41,6 +42,7 @@ User-Agent: UnicödeBöt
Disallow: /some/randome/page.html
""".encode('utf-8')
response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
+
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
@@ -77,6 +79,7 @@ Disallow: /some/randome/page.html
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')
+
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
@@ -99,6 +102,7 @@ Disallow: /some/randome/page.html
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
response = Response('http://site.local/robots.txt')
+
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
@@ -118,6 +122,7 @@ Disallow: /some/randome/page.html
def test_robotstxt_error(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
err = error.DNSLookupError('Robotstxt address not found')
+
def return_failure(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.errback, failure.Failure(err))
@@ -133,6 +138,7 @@ Disallow: /some/randome/page.html
def test_robotstxt_immediate_error(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
err = error.DNSLookupError('Robotstxt address not found')
+
def immediate_failure(request, spider):
deferred = Deferred()
deferred.errback(failure.Failure(err))
@@ -144,6 +150,7 @@ Disallow: /some/randome/page.html
def test_ignore_robotstxt_request(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+
def ignore_request(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))
@@ -167,3 +174,21 @@ Disallow: /some/randome/page.html
spider = None # not actually used
return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),
IgnoreRequest)
+
+
+class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
+ if not rerp_available():
+ skip = "Rerp parser is not installed"
+
+ def setUp(self):
+ super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
+ self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
+
+
+class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
+ if not reppy_available():
+ skip = "Reppy parser is not installed"
+
+ def setUp(self):
+ super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
+ self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
new file mode 100644
index 000000000..2819786b5
--- /dev/null
+++ b/tests/test_robotstxt_interface.py
@@ -0,0 +1,142 @@
+# coding=utf-8
+from twisted.trial import unittest
+from scrapy.utils.python import to_native_str
+
+
+def reppy_available():
+ # check if reppy parser is installed
+ try:
+ from reppy.robots import Robots
+ except ImportError:
+ return False
+ return True
+
+
+def rerp_available():
+ # check if robotexclusionrulesparser is installed
+ try:
+ from robotexclusionrulesparser import RobotExclusionRulesParser
+ except ImportError:
+ return False
+ return True
+
+
+class BaseRobotParserTest:
+ def _setUp(self, parser_cls):
+ self.parser_cls = parser_cls
+
+ def test_allowed(self):
+ robotstxt_robotstxt_body = ("User-agent: * \n"
+ "Disallow: /disallowed \n"
+ "Allow: /allowed \n"
+ "Crawl-delay: 10".encode('utf-8'))
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+ self.assertTrue(rp.allowed("https://www.site.local/allowed", "*"))
+ self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
+
+ def test_allowed_wildcards(self):
+ robotstxt_robotstxt_body = """User-agent: first
+ Disallow: /disallowed/*/end$
+
+ User-agent: second
+ Allow: /*allowed
+ Disallow: /
+ """.encode('utf-8')
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+
+ self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first"))
+ self.assertFalse(rp.allowed("https://www.site.local/disallowed/xyz/end", "first"))
+ self.assertFalse(rp.allowed("https://www.site.local/disallowed/abc/end", "first"))
+ self.assertTrue(rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first"))
+
+ self.assertTrue(rp.allowed("https://www.site.local/allowed", "second"))
+ self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second"))
+ self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
+
+ def test_length_based_precedence(self):
+ robotstxt_robotstxt_body = ("User-agent: * \n"
+ "Disallow: / \n"
+ "Allow: /page".encode('utf-8'))
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+ self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
+
+ def test_order_based_precedence(self):
+ robotstxt_robotstxt_body = ("User-agent: * \n"
+ "Disallow: / \n"
+ "Allow: /page".encode('utf-8'))
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+ self.assertFalse(rp.allowed("https://www.site.local/page", "*"))
+
+ def test_empty_response(self):
+ """empty response should equal 'allow all'"""
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b'')
+ self.assertTrue(rp.allowed("https://site.local/", "*"))
+ self.assertTrue(rp.allowed("https://site.local/", "chrome"))
+ self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
+ self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
+
+ def test_garbage_response(self):
+ """garbage response should be discarded, equal 'allow all'"""
+ robotstxt_robotstxt_body = b'GIF89a\xd3\x00\xfe\x00\xa2'
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+ self.assertTrue(rp.allowed("https://site.local/", "*"))
+ self.assertTrue(rp.allowed("https://site.local/", "chrome"))
+ self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
+ self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
+
+ def test_unicode_url_and_useragent(self):
+ robotstxt_robotstxt_body = u"""
+ User-Agent: *
+ Disallow: /admin/
+ Disallow: /static/
+ # taken from https://en.wikipedia.org/robots.txt
+ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+ Disallow: /wiki/Käyttäjä:
+
+ User-Agent: UnicödeBöt
+ Disallow: /some/randome/page.html""".encode('utf-8')
+ rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+ self.assertTrue(rp.allowed("https://site.local/", "*"))
+ self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
+ self.assertFalse(rp.allowed("https://site.local/static/", "*"))
+ self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+ self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
+ self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+ self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
+ self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+
+
+class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+ def setUp(self):
+ from scrapy.robotstxt import PythonRobotParser
+ super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
+
+ def test_length_based_precedence(self):
+ raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
+
+ def test_allowed_wildcards(self):
+ raise unittest.SkipTest("RobotFileParser does not support wildcards.")
+
+
+class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+ if not reppy_available():
+ skip = "Reppy parser is not installed"
+
+ def setUp(self):
+ from scrapy.robotstxt import ReppyRobotParser
+ super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
+
+ def test_order_based_precedence(self):
+ raise unittest.SkipTest("Rerp does not support order based directives precedence.")
+
+
+class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+ if not rerp_available():
+ skip = "Rerp parser is not installed"
+
+ def setUp(self):
+ from scrapy.robotstxt import RerpRobotParser
+ super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
+
+ def test_length_based_precedence(self):
+ raise unittest.SkipTest("Rerp does not support length based directives precedence.")
diff --git a/tox.ini b/tox.ini
index 157a8b3ed..c918731f4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -116,3 +116,17 @@ changedir = {[docs]changedir}
deps = {[docs]deps}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+[testenv:py37-extra-deps]
+basepython = python3.7
+deps =
+ {[testenv:py34]deps}
+ reppy
+ robotexclusionrulesparser
+
+[testenv:py27-extra-deps]
+basepython = python2.7
+deps =
+ {[testenv]deps}
+ reppy
+ robotexclusionrulesparser