Scrapy, a fast high-level web crawling & scraping framework for Python.
Add from_crawler() to components that only had from_settings().
Andrey Rakhmatullin499e7e8aa685
diff --git a/docs/topics/email.rst b/docs/topics/email.rstindex d6a7ad354..8f7a2357a 100644--- a/docs/topics/email.rst+++ b/docs/topics/email.rst@@ -27,13 +27,13 @@ the standard ``__init__`` method:mailer = MailSender()-Or you can instantiate it passing a Scrapy settings object, which will respect-the :ref:`settings <topics-email-settings>`:+Or you can instantiate it passing a :class:`scrapy.Crawler` instance, which+will respect the :ref:`settings <topics-email-settings>`:.. skip: start.. code-block:: python- mailer = MailSender.from_settings(settings)+ mailer = MailSender.from_crawler(crawler)And here is how to use it to send an e-mail (without attachments):@@ -81,13 +81,13 @@ rest of the framework.:param smtpssl: enforce using a secure SSL connection:type smtpssl: bool- .. classmethod:: from_settings(settings)+ .. classmethod:: from_crawler(crawler)- Instantiate using a Scrapy settings object, which will respect- :ref:`these Scrapy settings <topics-email-settings>`.+ Instantiate using a :class:`scrapy.Crawler` instance, which will+ respect :ref:`these Scrapy settings <topics-email-settings>`.- :param settings: the e-mail recipients- :type settings: :class:`scrapy.settings.Settings` object+ :param crawler: the crawler+ :type settings: :class:`scrapy.Crawler` object.. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None)diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 7c15b67e8..710e2e131 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -488,7 +488,7 @@ A request fingerprinter is a class that must implement the following method::param request: request to fingerprint:type request: scrapy.http.Request-Additionally, it may also implement the following methods:+Additionally, it may also implement the following method:.. classmethod:: from_crawler(cls, crawler):noindex:@@ -504,13 +504,6 @@ Additionally, it may also implement the following methods::param crawler: crawler that uses this request fingerprinter:type crawler: :class:`~scrapy.crawler.Crawler` object-.. classmethod:: from_settings(cls, settings)-- If present, and ``from_crawler`` is not defined, this class method is called- to create a request fingerprinter instance from a- :class:`~scrapy.settings.Settings` object. It must return a new instance of- the request fingerprinter.-.. currentmodule:: scrapy.httpThe :meth:`fingerprint` method of the default request fingerprinter,diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex f80f832a7..8e17eab9a 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -21,6 +21,7 @@ from scrapy.core.downloader.tls import (ScrapyClientTLSOptions,openssl_methods,)+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:@@ -69,6 +70,31 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):method: int = SSL.SSLv23_METHOD,*args: Any,**kwargs: Any,+ ) -> Self:+ warnings.warn(+ f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return cls._from_settings(settings, method, *args, **kwargs)++ @classmethod+ def from_crawler(+ cls,+ crawler: Crawler,+ method: int = SSL.SSLv23_METHOD,+ *args: Any,+ **kwargs: Any,+ ) -> Self:+ return cls._from_settings(crawler.settings, method, *args, **kwargs)++ @classmethod+ def _from_settings(+ cls,+ settings: BaseSettings,+ method: int = SSL.SSLv23_METHOD,+ *args: Any,+ **kwargs: Any,) -> Self:tls_verbose_logging: bool = settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex d37d2741a..7b8eea135 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -1,9 +1,11 @@from __future__ import annotationsimport logging+import warningsfrom pathlib import Pathfrom typing import TYPE_CHECKING+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.job import job_dirfrom scrapy.utils.request import (RequestFingerprinter,@@ -26,6 +28,15 @@ if TYPE_CHECKING:class BaseDupeFilter:@classmethoddef from_settings(cls, settings: BaseSettings) -> Self:+ warnings.warn(+ f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return cls()++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:return cls()def request_seen(self, request: Request) -> bool:@@ -72,17 +83,31 @@ class RFPDupeFilter(BaseDupeFilter):*,fingerprinter: RequestFingerprinterProtocol | None = None,) -> Self:- debug = settings.getbool("DUPEFILTER_DEBUG")- return cls(job_dir(settings), debug, fingerprinter=fingerprinter)+ warnings.warn(+ f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return cls._from_settings(settings, fingerprinter=fingerprinter)@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:assert crawler.request_fingerprinter- return cls.from_settings(+ return cls._from_settings(crawler.settings,fingerprinter=crawler.request_fingerprinter,)+ @classmethod+ def _from_settings(+ cls,+ settings: BaseSettings,+ *,+ fingerprinter: RequestFingerprinterProtocol | None = None,+ ) -> Self:+ debug = settings.getbool("DUPEFILTER_DEBUG")+ return cls(job_dir(settings), debug, fingerprinter=fingerprinter)+def request_seen(self, request: Request) -> bool:fp = self.request_fingerprint(request)if fp in self.fingerprints:diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex 73d864d5d..d7f810107 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -48,7 +48,7 @@ class MemoryUsage:self.check_interval: float = crawler.settings.getfloat("MEMUSAGE_CHECK_INTERVAL_SECONDS")- self.mail: MailSender = MailSender.from_settings(crawler.settings)+ self.mail: MailSender = MailSender.from_crawler(crawler)crawler.signals.connect(self.engine_started, signal=signals.engine_started)crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.pyindex 600eebcf2..221628642 100644--- a/scrapy/extensions/statsmailer.py+++ b/scrapy/extensions/statsmailer.py@@ -33,7 +33,7 @@ class StatsMailer:recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS")if not recipients:raise NotConfigured- mail: MailSender = MailSender.from_settings(crawler.settings)+ mail: MailSender = MailSender.from_crawler(crawler)assert crawler.statso = cls(crawler.stats, recipients, mail)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)diff --git a/scrapy/mail.py b/scrapy/mail.pyindex ce7beb773..3c40fea34 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -7,6 +7,7 @@ See documentation in docs/topics/email.rstfrom __future__ import annotationsimport logging+import warningsfrom email import encoders as Encodersfrom email.mime.base import MIMEBasefrom email.mime.multipart import MIMEMultipart@@ -19,6 +20,7 @@ from typing import IO, TYPE_CHECKING, Anyfrom twisted.internet import sslfrom twisted.internet.defer import Deferred+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import to_bytes@@ -32,6 +34,7 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ from scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings@@ -72,6 +75,19 @@ class MailSender:@classmethoddef from_settings(cls, settings: BaseSettings) -> Self:+ warnings.warn(+ f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return cls._from_settings(settings)++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ return cls._from_settings(crawler.settings)++ @classmethod+ def _from_settings(cls, settings: BaseSettings) -> Self:return cls(smtphost=settings["MAIL_HOST"],mailfrom=settings["MAIL_FROM"],diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.pyindex 191adb6cd..a1cd1bb7c 100644--- a/scrapy/spidermiddlewares/urllength.py+++ b/scrapy/spidermiddlewares/urllength.py@@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rstfrom __future__ import annotationsimport logging+import warningsfrom typing import TYPE_CHECKING, Any-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responseif TYPE_CHECKING:@@ -19,6 +20,7 @@ if TYPE_CHECKING:from typing_extensions import Selffrom scrapy import Spider+ from scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings@@ -31,6 +33,19 @@ class UrlLengthMiddleware:@classmethoddef from_settings(cls, settings: BaseSettings) -> Self:+ warnings.warn(+ f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return cls._from_settings(settings)++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ return cls._from_settings(crawler.settings)++ @classmethod+ def _from_settings(cls, settings: BaseSettings) -> Self:maxlength = settings.getint("URLLENGTH_LIMIT")if not maxlength:raise NotConfigureddiff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 9ba8bd64f..4fd648f48 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -33,14 +33,6 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):return df-class FromSettingsRFPDupeFilter(RFPDupeFilter):- @classmethod- def from_settings(cls, settings, *, fingerprinter=None):- df = super().from_settings(settings, fingerprinter=fingerprinter)- df.method = "from_settings"- return df--class DirectDupeFilter:method = "n/a"@@ -56,16 +48,6 @@ class RFPDupeFilterTest(unittest.TestCase):self.assertTrue(scheduler.df.debug)self.assertEqual(scheduler.df.method, "from_crawler")- def test_df_from_settings_scheduler(self):- settings = {- "DUPEFILTER_DEBUG": True,- "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter,- }- crawler = get_crawler(settings_dict=settings)- scheduler = Scheduler.from_crawler(crawler)- self.assertTrue(scheduler.df.debug)- self.assertEqual(scheduler.df.method, "from_settings")-def test_df_direct_scheduler(self):settings = {"DUPEFILTER_CLASS": DirectDupeFilter,diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex 9111e4c82..1a0f2e223 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -3,7 +3,6 @@ from unittest import TestCasefrom testfixtures import LogCapturefrom scrapy.http import Request, Response-from scrapy.settings import Settingsfrom scrapy.spidermiddlewares.urllength import UrlLengthMiddlewarefrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler@@ -12,12 +11,10 @@ from scrapy.utils.test import get_crawlerclass TestUrlLengthMiddleware(TestCase):def setUp(self):self.maxlength = 25- settings = Settings({"URLLENGTH_LIMIT": self.maxlength})-- crawler = get_crawler(Spider)+ crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength})self.spider = crawler._create_spider("foo")self.stats = crawler.stats- self.mw = UrlLengthMiddleware.from_settings(settings)+ self.mw = UrlLengthMiddleware.from_crawler(crawler)self.response = Response("http://scrapytest.org")self.short_url_req = Request("http://scrapytest.org/")