Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate ReppyRobotParser (#6099)
Andrey Rakhmatullin39ee8d1ee2de
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex a8e5b23bf..1abbc4968 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -1039,8 +1039,8 @@ RobotsTxtMiddleware* :ref:`Protego <protego-parser>` (default)* :ref:`RobotFileParser <python-robotfileparser>`- * :ref:`Reppy <reppy-parser>`* :ref:`Robotexclusionrulesparser <rerp-parser>`+ * :ref:`Reppy <reppy-parser>` (deprecated)You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.@@ -1133,6 +1133,7 @@ In order to use this parser:.. warning:: `Upstream issue #122<https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.+ Because of this the Reppy parser is deprecated.* Set :setting:`ROBOTSTXT_PARSER` setting to``scrapy.robotstxt.ReppyRobotParser``diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 604b5e314..5c5ac4e41 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -1,7 +1,9 @@import loggingimport sysfrom abc import ABCMeta, abstractmethod+from warnings import warn+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import to_unicodelogger = logging.getLogger(__name__)@@ -79,6 +81,7 @@ class PythonRobotParser(RobotParser):class ReppyRobotParser(RobotParser):def __init__(self, robotstxt_body, spider):+ warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)from reppy.robots import Robotsself.spider = spider