Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove build_from_settings().
Andrey Rakhmatullineda3a89b3fe3
diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex b6a427895..9e994703d 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -2,12 +2,13 @@ from __future__ import annotationsimport loggingimport pprint+import warningsfrom collections import defaultdict, dequefrom typing import TYPE_CHECKING, Any, TypeVar, cast-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.utils.defer import process_chain, process_parallel-from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object+from scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:from collections.abc import Callable, Iterable@@ -20,7 +21,7 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawler- from scrapy.settings import Settings+ from scrapy.settings import BaseSettings, Settings_P = ParamSpec("_P")@@ -50,8 +51,27 @@ class MiddlewareManager:def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:raise NotImplementedError+ @staticmethod+ def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T:+ if hasattr(objcls, "from_settings"):+ instance = objcls.from_settings(settings) # type: ignore[attr-defined]+ method_name = "from_settings"+ else:+ instance = objcls()+ method_name = "__new__"+ if instance is None:+ raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")+ return cast(_T, instance)+@classmethoddef from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:+ if crawler is None:+ warnings.warn(+ "Calling MiddlewareManager.from_settings() without a Crawler instance is deprecated."+ " As this method will be deprecated in the future, please switch to from_crawler().",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )mwlist = cls._get_mwlist_from_settings(settings)middlewares = []enabled = []@@ -61,7 +81,7 @@ class MiddlewareManager:if crawler is not None:mw = build_from_crawler(mwcls, crawler)else:- mw = build_from_settings(mwcls, settings)+ mw = MiddlewareManager._build_from_settings(mwcls, settings)middlewares.append(mw)enabled.append(clspath)except NotConfigured as e:diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 12c09839f..efb475131 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -26,7 +26,6 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettings_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes@@ -150,7 +149,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):"""warnings.warn("The create_instance() function is deprecated. "- "Please use build_from_crawler() or build_from_settings() instead.",+ "Please use build_from_crawler() instead.",category=ScrapyDeprecationWarning,stacklevel=2,)@@ -176,7 +175,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):def build_from_crawler(objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any) -> T:- """Construct a class instance using its ``from_crawler`` constructor.+ """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor.``*args`` and ``**kwargs`` are forwarded to the constructor.@@ -196,26 +195,6 @@ def build_from_crawler(return cast(T, instance)-def build_from_settings(- objcls: type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any-) -> T:- """Construct a class instance using its ``from_settings`` constructor.-- ``*args`` and ``**kwargs`` are forwarded to the constructor.-- Raises ``TypeError`` if the resulting instance is ``None``.- """- if hasattr(objcls, "from_settings"):- instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined]- method_name = "from_settings"- else:- instance = objcls(*args, **kwargs)- method_name = "__new__"- if instance is None:- raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return cast(T, instance)--@contextmanagerdef set_environ(**kwargs: str) -> Iterator[None]:"""Temporarily set environment variables inside the context manager anddiff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex a42c7b3d1..3a1cf19ad 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -2,7 +2,7 @@ from twisted.trial import unittestfrom scrapy.exceptions import NotConfiguredfrom scrapy.middleware import MiddlewareManager-from scrapy.settings import Settings+from scrapy.utils.test import get_crawlerclass M1:@@ -23,8 +23,6 @@ class M2:def close_spider(self, spider):pass- pass-class M3:def process(self, response, request, spider):@@ -83,7 +81,7 @@ class MiddlewareManagerTest(unittest.TestCase):self.assertEqual(mwman.middlewares, (m1, m2, m3))def test_enabled_from_settings(self):- settings = Settings()- mwman = TestMiddlewareManager.from_settings(settings)+ crawler = get_crawler()+ mwman = TestMiddlewareManager.from_crawler(crawler)classes = [x.__class__ for x in mwman.middlewares]self.assertEqual(classes, [M1, M3])diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex 4d8e71521..f71b2b034 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -10,7 +10,6 @@ from scrapy.item import Field, Itemfrom scrapy.utils.misc import (arg_to_iter,build_from_crawler,- build_from_settings,create_instance,load_object,rel_has_nofollow,@@ -197,39 +196,6 @@ class UtilsMiscTestCase(unittest.TestCase):with self.assertRaises(TypeError):build_from_crawler(m, crawler, *args, **kwargs)- def test_build_from_settings(self):- settings = mock.MagicMock()- args = (True, 100.0)- kwargs = {"key": "val"}-- def _test_with_settings(mock, settings):- build_from_settings(mock, settings, *args, **kwargs)- if hasattr(mock, "from_settings"):- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)- self.assertEqual(mock.call_count, 0)- else:- mock.assert_called_once_with(*args, **kwargs)-- # Check usage of correct constructor using three mocks:- # 1. with no alternative constructors- # 2. with from_settings() constructor- # 3. with from_settings() and from_crawler() constructor- spec_sets = (- ["__qualname__"],- ["__qualname__", "from_settings"],- ["__qualname__", "from_settings", "from_crawler"],- )- for specs in spec_sets:- m = mock.MagicMock(spec_set=specs)- _test_with_settings(m, settings)- m.reset_mock()-- # Check adoption of crawler settings- m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])- m.from_settings.return_value = None- with self.assertRaises(TypeError):- build_from_settings(m, settings, *args, **kwargs)-def test_set_environ(self):assert os.environ.get("some_test_environ") is Nonewith set_environ(some_test_environ="test_value"):diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex cce119001..1797d5e1f 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -9,25 +9,18 @@ from tempfile import mkdtempimport OpenSSL.SSLfrom twisted.internet import defer, reactor-from twisted.trial import unittest-from twisted.web import resource, server, static, util--try:- from twisted.internet.testing import StringTransport-except ImportError:- # deprecated in Twisted 19.7.0- # (remove once we bump our requirement past that version)- from twisted.test.proto_helpers import StringTransport-from twisted.internet.defer import inlineCallbacks+from twisted.internet.testing import StringTransportfrom twisted.protocols.policies import WrappingFactory+from twisted.trial import unittest+from twisted.web import resource, server, static, utilfrom scrapy.core.downloader import webclient as clientfrom scrapy.core.downloader.contextfactory import ScrapyClientContextFactoryfrom scrapy.http import Headers, Request-from scrapy.settings import Settings-from scrapy.utils.misc import build_from_settings+from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytes, to_unicode+from scrapy.utils.test import get_crawlerfrom tests.mockserver import (BrokenDownloadResource,ErrorResource,@@ -469,22 +462,22 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):def testPayload(self):s = "0123456789" * 10- settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers})- client_context_factory = build_from_settings(- ScrapyClientContextFactory, settings+ crawler = get_crawler(+ settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers})+ client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler)return getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))def testPayloadDisabledCipher(self):s = "0123456789" * 10- settings = Settings(- {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"}- )- client_context_factory = build_from_settings(- ScrapyClientContextFactory, settings+ crawler = get_crawler(+ settings_dict={+ "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"+ })+ client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler)d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory)