Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Drop Reppy.

Andrey Rakhmatullin5759b3f0f2b0
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index c31f7fe43..13064ccdd 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1086,7 +1086,6 @@ RobotsTxtMiddleware
* :ref:`Protego <protego-parser>` (default)
* :ref:`RobotFileParser <python-robotfileparser>`
* :ref:`Robotexclusionrulesparser <rerp-parser>`
- * :ref:`Reppy <reppy-parser>` (deprecated)
You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@@ -1154,37 +1153,6 @@ In order to use this parser, set:
* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
-.. _reppy-parser:
-
-Reppy parser
-~~~~~~~~~~~~
-
-Based on `Reppy <https://github.com/seomoz/reppy/>`_:
-
-* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
- <https://github.com/seomoz/rep-cpp>`_
-
-* is compliant with `Martijn Koster's 1996 draft specification
- <https://www.robotstxt.org/norobots-rfc.txt>`_
-
-* supports wildcard matching
-
-* uses the length based rule
-
-Native implementation, provides better speed than Protego.
-
-In order to use this parser:
-
-* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
-
- .. warning:: `Upstream issue #122
- <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
- Because of this the Reppy parser is deprecated.
-
-* Set :setting:`ROBOTSTXT_PARSER` setting to
- ``scrapy.robotstxt.ReppyRobotParser``
-
-
.. _rerp-parser:
Robotexclusionrulesparser
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index a0e5fc671..f0a6e7467 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -4,9 +4,7 @@ import logging
import sys
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING
-from warnings import warn
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
@@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser):
return self.rp.can_fetch(user_agent, url)
-class ReppyRobotParser(RobotParser):
- def __init__(self, robotstxt_body: bytes, spider: Spider | None):
- warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
- from reppy.robots import Robots
-
- self.spider: Spider | None = spider
- self.rp = Robots.parse("", robotstxt_body)
-
- @classmethod
- def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
- spider = None if not crawler else crawler.spider
- o = cls(robotstxt_body, spider)
- return o
-
- def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
- return self.rp.allowed(url, user_agent)
-
-
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
from robotexclusionrulesparser import RobotExclusionRulesParser
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index e166cc000..12b541456 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
-from tests.test_robotstxt_interface import reppy_available, rerp_available
+from tests.test_robotstxt_interface import rerp_available
class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
self.crawler.settings.set(
"ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser"
)
-
-
-class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
- if not reppy_available():
- skip = "Reppy parser is not installed"
-
- def setUp(self):
- super().setUp()
- self.crawler.settings.set(
- "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser"
- )
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 28ad910a8..541979dcc 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -3,15 +3,6 @@ from twisted.trial import unittest
from scrapy.robotstxt import decode_robotstxt
-def reppy_available():
- # check if reppy parser is installed
- try:
- from reppy.robots import Robots # noqa: F401
- except ImportError:
- return False
- return True
-
-
def rerp_available():
# check if robotexclusionrulesparser is installed
try:
@@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
raise unittest.SkipTest("RobotFileParser does not support wildcards.")
-class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
- if not reppy_available():
- skip = "Reppy parser is not installed"
-
- def setUp(self):
- from scrapy.robotstxt import ReppyRobotParser
-
- super()._setUp(ReppyRobotParser)
-
- def test_order_based_precedence(self):
- raise unittest.SkipTest(
- "Reppy does not support order based directives precedence."
- )
-
-
class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
if not rerp_available():
skip = "Rerp parser is not installed"