Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop Reppy.
Andrey Rakhmatullin5759b3f0f2b0
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex c31f7fe43..13064ccdd 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -1086,7 +1086,6 @@ RobotsTxtMiddleware* :ref:`Protego <protego-parser>` (default)* :ref:`RobotFileParser <python-robotfileparser>`* :ref:`Robotexclusionrulesparser <rerp-parser>`- * :ref:`Reppy <reppy-parser>` (deprecated)You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.@@ -1154,37 +1153,6 @@ In order to use this parser, set:* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``-.. _reppy-parser:--Reppy parser-~~~~~~~~~~~~--Based on `Reppy <https://github.com/seomoz/reppy/>`_:--* is a Python wrapper around `Robots Exclusion Protocol Parser for C++- <https://github.com/seomoz/rep-cpp>`_--* is compliant with `Martijn Koster's 1996 draft specification- <https://www.robotstxt.org/norobots-rfc.txt>`_--* supports wildcard matching--* uses the length based rule--Native implementation, provides better speed than Protego.--In order to use this parser:--* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``-- .. warning:: `Upstream issue #122- <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.- Because of this the Reppy parser is deprecated.--* Set :setting:`ROBOTSTXT_PARSER` setting to- ``scrapy.robotstxt.ReppyRobotParser``--.. _rerp-parser:Robotexclusionrulesparserdiff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex a0e5fc671..f0a6e7467 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -4,9 +4,7 @@ import loggingimport sysfrom abc import ABCMeta, abstractmethodfrom typing import TYPE_CHECKING-from warnings import warn-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:@@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser):return self.rp.can_fetch(user_agent, url)-class ReppyRobotParser(RobotParser):- def __init__(self, robotstxt_body: bytes, spider: Spider | None):- warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)- from reppy.robots import Robots-- self.spider: Spider | None = spider- self.rp = Robots.parse("", robotstxt_body)-- @classmethod- def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:- spider = None if not crawler else crawler.spider- o = cls(robotstxt_body, spider)- return o-- def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:- return self.rp.allowed(url, user_agent)--class RerpRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):from robotexclusionrulesparser import RobotExclusionRulesParserdiff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex e166cc000..12b541456 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.settings import Settings-from tests.test_robotstxt_interface import reppy_available, rerp_available+from tests.test_robotstxt_interface import rerp_availableclass RobotsTxtMiddlewareTest(unittest.TestCase):@@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):self.crawler.settings.set("ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser")---class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):- if not reppy_available():- skip = "Reppy parser is not installed"-- def setUp(self):- super().setUp()- self.crawler.settings.set(- "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser"- )diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 28ad910a8..541979dcc 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -3,15 +3,6 @@ from twisted.trial import unittestfrom scrapy.robotstxt import decode_robotstxt-def reppy_available():- # check if reppy parser is installed- try:- from reppy.robots import Robots # noqa: F401- except ImportError:- return False- return True--def rerp_available():# check if robotexclusionrulesparser is installedtry:@@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):raise unittest.SkipTest("RobotFileParser does not support wildcards.")-class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):- if not reppy_available():- skip = "Reppy parser is not installed"-- def setUp(self):- from scrapy.robotstxt import ReppyRobotParser-- super()._setUp(ReppyRobotParser)-- def test_order_based_precedence(self):- raise unittest.SkipTest(- "Reppy does not support order based directives precedence."- )--class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):if not rerp_available():skip = "Rerp parser is not installed"