Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove code deprecated in 2.12.0 (#7126)
Andrey Rakhmatullin61b4befc60cf
diff --git a/docs/conf.py b/docs/conf.pyindex 5c5926dbe..9b1db9bb7 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -126,7 +126,7 @@ coverage_ignore_pyobjects = [# The interface methods of duplicate request filtering classes are already# covered in the interface documentation part of the DUPEFILTER_CLASS# setting documentation.- r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",+ r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$",# Private exception used by the command-line interface implementation.r"^scrapy\.exceptions\.UsageError",# Methods of BaseItemExporter subclasses are only documented indiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 5e56de05c..a92070918 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1131,9 +1131,9 @@ interface::class MyDupeFilter:@classmethod- def from_settings(cls, settings):+ def from_crawler(cls, crawler):"""Returns an instance of this duplicate request filtering class- based on the current crawl settings."""+ based on the current Crawler instance."""return cls()def request_seen(self, request):diff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex 280a0f659..c5b6483e3 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -29,23 +29,6 @@ __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))-def __getattr__(name: str):- if name == "twisted_version":- import warnings # noqa: PLC0415 # pylint: disable=reimported-- from twisted import version as _txv # noqa: PLC0415-- from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415-- warnings.warn(- "The scrapy.twisted_version attribute is deprecated, use twisted.version instead",- ScrapyDeprecationWarning,- )- return _txv.major, _txv.minor, _txv.micro-- raise AttributeError--# Ignore noisy twisted deprecation warningswarnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 76cd376e1..77b17287d 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -182,14 +182,6 @@ class Downloader:return key- def _get_slot_key(self, request: Request, spider: Spider | None) -> str:- warnings.warn(- "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",- ScrapyDeprecationWarning,- stacklevel=2,- )- return self.get_slot_key(request)-# passed as download_func into self.middleware.download() in self.fetch()@inlineCallbacksdef _enqueue_request(diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex d1ba6208a..5f7c9a8c3 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -71,21 +71,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):stacklevel=2,)- @classmethod- def from_settings(- cls,- settings: BaseSettings,- method: int = SSL.SSLv23_METHOD,- *args: Any,- **kwargs: Any,- ) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls._from_settings(settings, method, *args, **kwargs)-@classmethoddef from_crawler(cls,@@ -94,20 +79,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):*args: Any,**kwargs: Any,) -> Self:- return cls._from_settings(crawler.settings, method, *args, **kwargs)-- @classmethod- def _from_settings(- cls,- settings: BaseSettings,- method: int = SSL.SSLv23_METHOD,- *args: Any,- **kwargs: Any,- ) -> Self:- tls_verbose_logging: bool = settings.getbool(+ tls_verbose_logging: bool = crawler.settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING")- tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]+ tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]return cls( # type: ignore[misc]method=method,tls_verbose_logging=tls_verbose_logging,diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 520ec3380..2e264ade3 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -1,7 +1,6 @@from __future__ import annotationsimport logging-import warningsfrom pathlib import Pathfrom typing import TYPE_CHECKINGfrom warnings import warn@@ -22,7 +21,6 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.http.request import Request- from scrapy.settings import BaseSettingsfrom scrapy.spiders import Spider@@ -30,15 +28,6 @@ class BaseDupeFilter:"""Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)that does not filter out any request."""- @classmethod- def from_settings(cls, settings: BaseSettings) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls()-@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:return cls()@@ -88,38 +77,16 @@ class RFPDupeFilter(BaseDupeFilter):self.file.seek(0)self.fingerprints.update(x.rstrip() for x in self.file)- @classmethod- def from_settings(- cls,- settings: BaseSettings,- *,- fingerprinter: RequestFingerprinterProtocol | None = None,- ) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls._from_settings(settings, fingerprinter=fingerprinter)-@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:assert crawler.request_fingerprinter- return cls._from_settings(- crawler.settings,+ debug = crawler.settings.getbool("DUPEFILTER_DEBUG")+ return cls(+ job_dir(crawler.settings),+ debug,fingerprinter=crawler.request_fingerprinter,)- @classmethod- def _from_settings(- cls,- settings: BaseSettings,- *,- fingerprinter: RequestFingerprinterProtocol | None = None,- ) -> Self:- debug = settings.getbool("DUPEFILTER_DEBUG")- return cls(job_dir(settings), debug, fingerprinter=fingerprinter)-def request_seen(self, request: Request) -> bool:fp = self.request_fingerprint(request)if fp in self.fingerprints:diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex a850b330e..adb6fdc33 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -16,7 +16,7 @@ from collections.abc import Callablefrom datetime import datetime, timezonefrom pathlib import Path, PureWindowsPathfrom tempfile import NamedTemporaryFile-from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast+from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, castfrom urllib.parse import unquote, urlparsefrom twisted.internet.defer import Deferred, DeferredList, maybeDeferred@@ -35,8 +35,6 @@ from scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import without_none_valuesif TYPE_CHECKING:- from collections.abc import Iterable-from _typeshed import OpenBinaryModefrom twisted.python.failure import Failure@@ -54,25 +52,6 @@ UriParamsCallableT: TypeAlias = Callable[[dict[str, Any], Spider], dict[str, Any] | None]-_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")---def build_storage(- builder: Callable[..., _StorageT],- uri: str,- *args: Any,- feed_options: dict[str, Any] | None = None,- preargs: Iterable[Any] = (),- **kwargs: Any,-) -> _StorageT:- warnings.warn(- "scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- kwargs["feed_options"] = feed_options- return builder(*preargs, uri, *args, **kwargs)-class ItemFilter:"""diff --git a/scrapy/mail.py b/scrapy/mail.pyindex 88612daa9..84209effa 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -7,7 +7,6 @@ See documentation in docs/topics/email.rstfrom __future__ import annotationsimport logging-import warningsfrom email import encoders as Encodersfrom email.mime.base import MIMEBasefrom email.mime.multipart import MIMEMultipart@@ -20,7 +19,6 @@ from typing import IO, TYPE_CHECKING, Anyfrom twisted.internet import sslfrom twisted.internet.defer import Deferred-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import to_bytes@@ -35,7 +33,6 @@ if TYPE_CHECKING:from typing_extensions import Selffrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettingslogger = logging.getLogger(__name__)@@ -73,21 +70,9 @@ class MailSender:self.mailfrom: str = mailfromself.debug: bool = debug- @classmethod- def from_settings(cls, settings: BaseSettings) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls._from_settings(settings)-@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- return cls._from_settings(crawler.settings)-- @classmethod- def _from_settings(cls, settings: BaseSettings) -> Self:+ settings = crawler.settingsreturn cls(smtphost=settings["MAIL_HOST"],mailfrom=settings["MAIL_FROM"],diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 32fae7cc0..21a2ebfa3 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -23,7 +23,7 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawler- from scrapy.settings import BaseSettings, Settings+ from scrapy.settings import Settingslogger = logging.getLogger(__name__)@@ -100,43 +100,15 @@ class MiddlewareManager(ABC):def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:raise NotImplementedError- @staticmethod- def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T:- if hasattr(objcls, "from_settings"):- instance = objcls.from_settings(settings) # type: ignore[attr-defined]- method_name = "from_settings"- else:- instance = objcls()- method_name = "__new__"- if instance is None:- raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return cast("_T", instance)-- @classmethod- def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls._from_settings(settings, crawler)-@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- return cls._from_settings(crawler.settings, crawler)-- @classmethod- def _from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:- mwlist = cls._get_mwlist_from_settings(settings)+ mwlist = cls._get_mwlist_from_settings(crawler.settings)middlewares = []enabled = []for clspath in mwlist:try:mwcls = load_object(clspath)- if crawler is not None:- mw = build_from_crawler(mwcls, crawler)- else:- mw = MiddlewareManager._build_from_settings(mwcls, settings)+ mw = build_from_crawler(mwcls, crawler)middlewares.append(mw)enabled.append(clspath)except NotConfigured as e:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 38f8537d6..25af0aa72 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -12,7 +12,6 @@ import hashlibimport loggingimport mimetypesimport time-import warningsfrom collections import defaultdictfrom contextlib import suppressfrom ftplib import FTP@@ -25,17 +24,15 @@ from itemadapter import ItemAdapterfrom twisted.internet.defer import Deferred, maybeDeferredfrom twisted.internet.threads import deferToThread-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline-from scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.datatypes import CaseInsensitiveDict-from scrapy.utils.deprecate import method_is_overriddenfrom scrapy.utils.ftp import ftp_store_filefrom scrapy.utils.log import failure_to_exc_info-from scrapy.utils.python import get_func_args, global_object_name, to_bytes+from scrapy.utils.python import to_bytesfrom scrapy.utils.request import referer_strif TYPE_CHECKING:@@ -49,6 +46,7 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawler+ from scrapy.settings import BaseSettingslogger = logging.getLogger(__name__)@@ -161,7 +159,7 @@ class S3FilesStore:AWS_USE_SSL = NoneAWS_VERIFY = None- POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings+ POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()HEADERS = {"Cache-Control": "max-age=172800",}@@ -280,7 +278,7 @@ class GCSFilesStore:CACHE_CONTROL = "max-age=172800"# The bucket's default object ACL will be applied to the object.- # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.+ # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_crawler().POLICY = Nonedef __init__(self, uri: str):@@ -446,9 +444,8 @@ class FilesPipeline(MediaPipeline):self,store_uri: str | PathLike[str],download_func: Callable[[Request, Spider], Response] | None = None,- settings: Settings | dict[str, Any] | None = None,*,- crawler: Crawler | None = None,+ crawler: Crawler,):if not (store_uri and (store_uri := _to_string(store_uri))):from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415@@ -461,18 +458,7 @@ class FilesPipeline(MediaPipeline):f"to enable {self.__class__.__name__}.")- if crawler is not None:- if settings is not None:- warnings.warn(- f"FilesPipeline.__init__() was called with a crawler instance and a settings instance"- f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"- f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- settings = crawler.settings- elif isinstance(settings, dict) or settings is None:- settings = Settings(settings)+ settings = crawler.settingscls_name = "FilesPipeline"self.store: FilesStoreProtocol = self._get_store(store_uri)resolve = functools.partial(@@ -490,51 +476,14 @@ class FilesPipeline(MediaPipeline):resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD)- super().__init__(- download_func=download_func,- settings=settings if not crawler else None,- crawler=crawler,- )-- @classmethod- def from_settings(cls, settings: Settings) -> Self:- warnings.warn(- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return cls._from_settings(settings, None)+ super().__init__(download_func=download_func, crawler=crawler)@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- if method_is_overridden(cls, FilesPipeline, "from_settings"):- warnings.warn(- f"{global_object_name(cls)} overrides FilesPipeline.from_settings()."- f" This method is deprecated and won't be called in future Scrapy versions,"- f" please update your code so that it overrides from_crawler() instead.",- category=ScrapyDeprecationWarning,- )- o = cls.from_settings(crawler.settings)- o._finish_init(crawler)- return o- return cls._from_settings(crawler.settings, crawler)-- @classmethod- def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:+ settings = crawler.settingscls._update_stores(settings)store_uri = settings["FILES_STORE"]- if "crawler" in get_func_args(cls.__init__):- o = cls(store_uri, crawler=crawler)- else:- o = cls(store_uri, settings=settings)- if crawler:- o._finish_init(crawler)- warnings.warn(- f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."- " This is deprecated and the argument will be required in future Scrapy versions.",- category=ScrapyDeprecationWarning,- )- return o+ return cls(store_uri, crawler=crawler)@classmethoddef _update_stores(cls, settings: BaseSettings) -> None:diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 5ac211dfb..687d29fa5 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -8,19 +8,17 @@ from __future__ import annotationsimport functoolsimport hashlib-import warningsfrom contextlib import suppressfrom io import BytesIOfrom typing import TYPE_CHECKING, Anyfrom itemadapter import ItemAdapter-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileException, FilesPipeline, _md5sum-from scrapy.settings import Settings-from scrapy.utils.python import get_func_args, global_object_name, to_bytes+from scrapy.utils.python import to_bytesif TYPE_CHECKING:from collections.abc import Callable, Iterable@@ -58,9 +56,8 @@ class ImagesPipeline(FilesPipeline):self,store_uri: str | PathLike[str],download_func: Callable[[Request, Spider], Response] | None = None,- settings: Settings | dict[str, Any] | None = None,*,- crawler: Crawler | None = None,+ crawler: Crawler,):try:from PIL import Image, ImageOps # noqa: PLC0415@@ -72,26 +69,9 @@ class ImagesPipeline(FilesPipeline):"ImagesPipeline requires installing Pillow 8.3.2 or later")- super().__init__(- store_uri,- settings=settings if not crawler else None,- download_func=download_func,- crawler=crawler,- )-- if crawler is not None:- if settings is not None:- warnings.warn(- f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance"- f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"- f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- settings = crawler.settings- elif isinstance(settings, dict) or settings is None:- settings = Settings(settings)+ super().__init__(store_uri, download_func=download_func, crawler=crawler)+ settings = crawler.settingsresolve = functools.partial(self._key_for_pipe,base_class_name="ImagesPipeline",@@ -121,21 +101,11 @@ class ImagesPipeline(FilesPipeline):)@classmethod- def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:+ def from_crawler(cls, crawler: Crawler) -> Self:+ settings = crawler.settingscls._update_stores(settings)store_uri = settings["IMAGES_STORE"]- if "crawler" in get_func_args(cls.__init__):- o = cls(store_uri, crawler=crawler)- else:- o = cls(store_uri, settings=settings)- if crawler:- o._finish_init(crawler)- warnings.warn(- f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."- " This is deprecated and the argument will be required in future Scrapy versions.",- category=ScrapyDeprecationWarning,- )- return o+ return cls(store_uri, crawler=crawler)def file_downloaded(self,diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex f8fc3687e..438e5e7d6 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -19,14 +19,13 @@ from twisted.python.versions import Versionfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.request import NO_CALLBACK, Request-from scrapy.settings import Settingsfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_corofrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iter-from scrapy.utils.python import get_func_args, global_object_name+from scrapy.utils.python import global_object_nameif TYPE_CHECKING:from collections.abc import Callable, Generator@@ -37,6 +36,7 @@ if TYPE_CHECKING:from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.http import Response+ from scrapy.settings import Settingsfrom scrapy.utils.request import RequestFingerprinterProtocol@@ -55,10 +55,6 @@ logger = logging.getLogger(__name__)class MediaPipeline(ABC):- crawler: Crawler- _fingerprinter: RequestFingerprinterProtocol- _modern_init = False-LOG_FAILED_RESULTS: bool = Trueclass SpiderInfo:@@ -73,24 +69,17 @@ class MediaPipeline(ABC):def __init__(self,download_func: Callable[[Request, Spider], Response] | None = None,- settings: Settings | dict[str, Any] | None = None,*,- crawler: Crawler | None = None,+ crawler: Crawler,):+ self.crawler: Crawler = crawler+ assert crawler.request_fingerprinter+ self._fingerprinter: RequestFingerprinterProtocol = (+ crawler.request_fingerprinter+ )self.download_func = download_func- if crawler is not None:- if settings is not None:- warnings.warn(- f"MediaPipeline.__init__() was called with a crawler instance and a settings instance"- f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"- f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- settings = crawler.settings- elif isinstance(settings, dict) or settings is None:- settings = Settings(settings)+ settings = crawler.settingsresolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline", settings=settings)@@ -99,27 +88,6 @@ class MediaPipeline(ABC):)self._handle_statuses(self.allow_redirects)- if crawler:- self._finish_init(crawler)- self._modern_init = True- else:- warnings.warn(- f"MediaPipeline.__init__() was called without the crawler argument"- f" when creating {global_object_name(self.__class__)}."- f" This is deprecated and the argument will be required in future Scrapy versions.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-- def _finish_init(self, crawler: Crawler) -> None:- # This was done in from_crawler() before 2.12, now it's done in __init__()- # if the crawler was passed to it and may be needed to be called in other- # deprecated code paths explicitly too. After the crawler argument of __init__()- # becomes mandatory this should be inlined there.- self.crawler = crawler- assert crawler.request_fingerprinter- self._fingerprinter = crawler.request_fingerprinter-def _handle_statuses(self, allow_redirects: bool) -> None:self.handle_httpstatus_list = Noneif allow_redirects:@@ -143,30 +111,7 @@ class MediaPipeline(ABC):@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- pipe: Self- if hasattr(cls, "from_settings"):- pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]- warnings.warn(- f"{global_object_name(cls)} has from_settings() and either doesn't have"- " from_crawler() or calls MediaPipeline.from_crawler() from it,"- " so from_settings() was used to create the instance of it."- " This is deprecated and calling from_settings() will be removed"- " in a future Scrapy version. Please move the initialization code into"- " from_crawler() or __init__().",- category=ScrapyDeprecationWarning,- )- elif "crawler" in get_func_args(cls.__init__):- pipe = cls(crawler=crawler)- else:- pipe = cls()- warnings.warn(- f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."- " This is deprecated and the argument will be required in future Scrapy versions.",- category=ScrapyDeprecationWarning,- )- if not pipe._modern_init:- pipe._finish_init(crawler)- return pipe+ return cls(crawler=crawler)@_warn_spider_argdef open_spider(self, spider: Spider | None = None) -> None:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 543e5c043..e66f78c47 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -152,7 +152,6 @@ __all__ = ["REFERER_ENABLED","REFERRER_POLICY","REQUEST_FINGERPRINTER_CLASS",- "REQUEST_FINGERPRINTER_IMPLEMENTATION","RETRY_ENABLED","RETRY_EXCEPTIONS","RETRY_HTTP_CODES",@@ -451,7 +450,6 @@ REFERER_ENABLED = TrueREFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"-REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"RETRY_ENABLED = TrueRETRY_EXCEPTIONS = [diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 39cd6e540..8ca3bc0bd 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -317,31 +317,6 @@ def process_chain(return d-def process_chain_both(- callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],- errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],- input: Any, # noqa: A002- *a: _P.args,- **kw: _P.kwargs,-) -> Deferred:- """Return a Deferred built by chaining the given callbacks and errbacks"""- warnings.warn(- "process_chain_both() is deprecated and will be removed in a future"- " Scrapy version.",- ScrapyDeprecationWarning,- stacklevel=2,- )- d: Deferred = Deferred()- for cb, eb in zip(callbacks, errbacks, strict=False):- d.addCallback(cb, *a, **kw)- d.addErrback(eb, *a, **kw)- if isinstance(input, failure.Failure):- d.errback(input)- else:- d.callback(input)- return d--def process_parallel(callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],input: _T, # noqa: A002diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 1acb06757..7994a2471 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -129,52 +129,10 @@ def rel_has_nofollow(rel: str | None) -> bool:return rel is not None and "nofollow" in rel.replace(",", " ").split()-def create_instance(objcls, settings, crawler, *args, **kwargs):- """Construct a class instance using its ``from_crawler`` or- ``from_settings`` constructors, if available.-- At least one of ``settings`` and ``crawler`` needs to be different from- ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.- If ``crawler`` is ``None``, only the ``from_settings`` constructor will be- tried.-- ``*args`` and ``**kwargs`` are forwarded to the constructors.-- Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.-- .. versionchanged:: 2.2- Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an- extension has not been implemented correctly).- """- warnings.warn(- "The create_instance() function is deprecated. "- "Please use build_from_crawler() instead.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-- if settings is None:- if crawler is None:- raise ValueError("Specify at least one of settings and crawler.")- settings = crawler.settings- if crawler and hasattr(objcls, "from_crawler"):- instance = objcls.from_crawler(crawler, *args, **kwargs)- method_name = "from_crawler"- elif hasattr(objcls, "from_settings"):- instance = objcls.from_settings(settings, *args, **kwargs)- method_name = "from_settings"- else:- instance = objcls(*args, **kwargs)- method_name = "__new__"- if instance is None:- raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return instance--def build_from_crawler(objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any) -> T:- """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor.+ """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor... versionadded:: 2.12@@ -185,17 +143,6 @@ def build_from_crawler(if hasattr(objcls, "from_crawler"):instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]method_name = "from_crawler"- elif hasattr(objcls, "from_settings"):- warnings.warn(- f"{objcls.__qualname__} has from_settings() but not from_crawler()."- " This is deprecated and calling from_settings() will be removed in a future"- " Scrapy version. You can implement a simple from_crawler() that calls"- " from_settings() with crawler.settings.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined]- method_name = "from_settings"else:instance = objcls(*args, **kwargs)method_name = "__new__"diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex d9a5007cc..ca4b4bf28 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -8,14 +8,12 @@ import gcimport inspectimport reimport sys-import warningsimport weakreffrom collections.abc import AsyncIterator, Iterable, Mappingfrom functools import partial, wrapsfrom itertools import chainfrom typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.asyncgen import as_async_generatorif TYPE_CHECKING:@@ -32,47 +30,6 @@ _VT = TypeVar("_VT")_P = ParamSpec("_P")-def flatten(x: Iterable[Any]) -> list[Any]:- """flatten(sequence) -> list-- Returns a single, flat list which contains all elements retrieved- from the sequence and all recursively contained sub-sequences- (iterables).-- Examples:- >>> [1, 2, [3,4], (5,6)]- [1, 2, [3, 4], (5, 6)]- >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])- [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]- >>> flatten(["foo", "bar"])- ['foo', 'bar']- >>> flatten(["foo", ["baz", 42], "bar"])- ['foo', 'baz', 42, 'bar']- """- warnings.warn(- "The flatten function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return list(iflatten(x))---def iflatten(x: Iterable[Any]) -> Iterable[Any]:- """iflatten(sequence) -> iterator-- Similar to ``.flatten()``, but returns iterator instead"""- warnings.warn(- "The iflatten function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- for el in x:- if is_listlike(el):- yield from iflatten(el)- else:- yield el--def is_listlike(x: Any) -> bool:""">>> is_listlike("foo")@@ -289,31 +246,6 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:return args, kwargs-def equal_attributes(- obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None-) -> bool:- """Compare two objects attributes"""- warnings.warn(- "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- # not attributes given return False by default- if not attributes:- return False-- temp1, temp2 = object(), object()- for attr in attributes:- # support callables like itemgetter- if callable(attr):- if attr(obj1) != attr(obj2):- return False- elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2):- return False- # all attributes equal- return True--@overloaddef without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ...diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 9c1161968..616ab661f 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -7,16 +7,13 @@ from __future__ import annotationsimport hashlibimport json-import warningsfrom typing import TYPE_CHECKING, Any, Protocolfrom urllib.parse import urlunparsefrom weakref import WeakKeyDictionary-from w3lib.http import basic_auth_headerfrom w3lib.url import canonicalize_urlfrom scrapy import Request, Spider-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import to_bytes, to_unicode@@ -120,41 +117,12 @@ class RequestFingerprinter:return cls(crawler)def __init__(self, crawler: Crawler | None = None):- if crawler:- implementation = crawler.settings.get(- "REQUEST_FINGERPRINTER_IMPLEMENTATION"- )- else:- implementation = "SENTINEL"-- if implementation != "SENTINEL":- message = (- "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"- "It will be removed in a future version of Scrapy."- )- warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)self._fingerprint = fingerprintdef fingerprint(self, request: Request) -> bytes:return self._fingerprint(request)-def request_authenticate(- request: Request,- username: str,- password: str,-) -> None:- """Authenticate the given request (in place) using the HTTP basic access- authentication mechanism (RFC 2617) and the given username and password- """- warnings.warn(- "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- request.headers["Authorization"] = basic_auth_header(username, password)--def request_httprepr(request: Request) -> bytes:"""Return the raw HTTP representation (as bytes) of the given request.This is provided only for reference since it's not the actual stream ofdiff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.pyindex bcfae0c00..5d06bbe30 100644--- a/scrapy/utils/serialize.py+++ b/scrapy/utils/serialize.py@@ -1,13 +1,11 @@import datetimeimport decimalimport json-import warningsfrom typing import Anyfrom itemadapter import ItemAdapter, is_itemfrom twisted.internet import defer-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Response@@ -35,13 +33,3 @@ class ScrapyJSONEncoder(json.JSONEncoder):if is_item(o):return ItemAdapter(o).asdict()return super().default(o)---class ScrapyJSONDecoder(json.JSONDecoder):- def __init__(self, *args, **kwargs):- warnings.warn(- "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- super().__init__(*args, **kwargs)diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 4c0764396..4e0356350 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -12,7 +12,7 @@ from importlib import import_modulefrom pathlib import Pathfrom posixpath import splitfrom typing import TYPE_CHECKING, Any, TypeVar, cast-from unittest import TestCase, mock+from unittest import mockfrom twisted.trial.unittest import SkipTestfrom twisted.web.client import Agent@@ -166,20 +166,6 @@ def get_testenv() -> dict[str, str]:return env-def assert_samelines(- testcase: TestCase, text1: str, text2: str, msg: str | None = None-) -> None:- """Asserts text1 and text2 have the same lines, ignoring differences in- line endings between platforms- """- warnings.warn(- "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009--def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:q: asyncio.Queue[_T] = asyncio.Queue()getter = q.get()diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 99d5cbb6d..966cb320e 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -756,37 +756,6 @@ class TestBuildFromCrawler:assert len(w) == 0assert pipe.store- def test_has_old_init(self):- class Pipeline(FilesPipeline):- def __init__(self, store_uri, download_func=None, settings=None):- super().__init__(store_uri, download_func, settings)- self._init_called = True-- with warnings.catch_warnings(record=True) as w:- pipe = Pipeline.from_crawler(self.crawler)- assert pipe.crawler == self.crawler- assert pipe._fingerprinter- assert len(w) == 2- assert pipe._init_called-- def test_has_from_settings(self):- class Pipeline(FilesPipeline):- _from_settings_called = False-- @classmethod- def from_settings(cls, settings):- o = super().from_settings(settings)- o._from_settings_called = True- return o-- with warnings.catch_warnings(record=True) as w:- pipe = Pipeline.from_crawler(self.crawler)- assert pipe.crawler == self.crawler- assert pipe._fingerprinter- assert len(w) == 3- assert pipe.store- assert pipe._from_settings_called-def test_has_from_crawler_and_init(self):class Pipeline(FilesPipeline):_from_crawler_called = Falsediff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex cc1d73349..9f174cc5f 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -418,84 +418,6 @@ class TestBuildFromCrawler:assert pipe._fingerprinterassert len(w) == 0- def test_has_old_init(self):- class Pipeline(UserDefinedPipeline):- def __init__(self):- super().__init__()- self._init_called = True-- with warnings.catch_warnings(record=True) as w:- pipe = Pipeline.from_crawler(self.crawler)- assert pipe.crawler == self.crawler- assert pipe._fingerprinter- assert len(w) == 2- assert pipe._init_called-- def test_has_from_settings(self):- class Pipeline(UserDefinedPipeline):- _from_settings_called = False-- @classmethod- def from_settings(cls, settings):- o = cls()- o._from_settings_called = True- return o-- with warnings.catch_warnings(record=True) as w:- pipe = Pipeline.from_crawler(self.crawler)- assert pipe.crawler == self.crawler- assert pipe._fingerprinter- assert len(w) == 2- assert pipe._from_settings_called-- def test_has_from_settings_and_from_crawler(self):- class Pipeline(UserDefinedPipeline):- _from_settings_called = False- _from_crawler_called = False-- @classmethod- def from_settings(cls, settings):- o = cls()- o._from_settings_called = True- return o-- @classmethod- def from_crawler(cls, crawler):- o = super().from_crawler(crawler)- o._from_crawler_called = True- return o-- with warnings.catch_warnings(record=True) as w:- pipe = Pipeline.from_crawler(self.crawler)- assert pipe.crawler == self.crawler- assert pipe._fingerprinter- assert len(w) == 2- assert pipe._from_settings_called- assert pipe._from_crawler_called-- def test_has_from_settings_and_init(self):- class Pipeline(UserDefinedPipeline):- _from_settings_called = False-- def __init__(self, store_uri, settings):- super().__init__()- self._init_called = True-- @classmethod- def from_settings(cls, settings):- store_uri = settings["FILES_STORE"]- o = cls(store_uri, settings=settings)- o._from_settings_called = True- return o-- with warnings.catch_warnings(record=True) as w:- pipe = Pipeline.from_crawler(self.crawler)- assert pipe.crawler == self.crawler- assert pipe._fingerprinter- assert len(w) == 2- assert pipe._from_settings_called- assert pipe._init_called-def test_has_from_crawler_and_init(self):class Pipeline(UserDefinedPipeline):_from_crawler_called = Falsediff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.pyindex fa6d11123..4c365859b 100644--- a/tests/test_scrapy__getattr__.py+++ b/tests/test_scrapy__getattr__.py@@ -1,20 +1,6 @@import warnings-def test_deprecated_twisted_version():- with warnings.catch_warnings(record=True) as warns:- from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module- twisted_version,- )-- assert twisted_version is not None- assert isinstance(twisted_version, tuple)- assert (- "The scrapy.twisted_version attribute is deprecated, use twisted.version instead"- in warns[0].message.args- )--def test_deprecated_concurrent_requests_per_ip_attribute():with warnings.catch_warnings(record=True) as warns:from scrapy.settings.default_settings import ( # noqa: PLC0415diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex b330819d9..610fc0ffe 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -9,7 +9,6 @@ from scrapy.item import Field, Itemfrom scrapy.utils.misc import (arg_to_iter,build_from_crawler,- create_instance,load_object,rel_has_nofollow,set_environ,@@ -97,98 +96,29 @@ class TestUtilsMisc:assert list(arg_to_iter({"a": 1})) == [{"a": 1}]assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")- def test_create_instance(self):- settings = mock.MagicMock()- crawler = mock.MagicMock(spec_set=["settings"])- args = (True, 100.0)- kwargs = {"key": "val"}-- def _test_with_settings(mock, settings):- create_instance(mock, settings, None, *args, **kwargs)- if hasattr(mock, "from_crawler"):- assert mock.from_crawler.call_count == 0- if hasattr(mock, "from_settings"):- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)- assert mock.call_count == 0- else:- mock.assert_called_once_with(*args, **kwargs)-- def _test_with_crawler(mock, settings, crawler):- create_instance(mock, settings, crawler, *args, **kwargs)- if hasattr(mock, "from_crawler"):- mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)- if hasattr(mock, "from_settings"):- assert mock.from_settings.call_count == 0- assert mock.call_count == 0- elif hasattr(mock, "from_settings"):- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)- assert mock.call_count == 0- else:- mock.assert_called_once_with(*args, **kwargs)-- # Check usage of correct constructor using four mocks:- # 1. with no alternative constructors- # 2. with from_settings() constructor- # 3. with from_crawler() constructor- # 4. with from_settings() and from_crawler() constructor- spec_sets = (- ["__qualname__"],- ["__qualname__", "from_settings"],- ["__qualname__", "from_crawler"],- ["__qualname__", "from_settings", "from_crawler"],- )- for specs in spec_sets:- m = mock.MagicMock(spec_set=specs)- _test_with_settings(m, settings)- m.reset_mock()- _test_with_crawler(m, settings, crawler)-- # Check adoption of crawler settings- m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])- create_instance(m, None, crawler, *args, **kwargs)- m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)-- with pytest.raises(- ValueError, match="Specify at least one of settings and crawler"- ):- create_instance(m, None, None)-- m.from_settings.return_value = None- with pytest.raises(TypeError):- create_instance(m, settings, None)-def test_build_from_crawler(self):- settings = mock.MagicMock()crawler = mock.MagicMock(spec_set=["settings"])args = (True, 100.0)kwargs = {"key": "val"}- def _test_with_crawler(mock, settings, crawler):+ def _test_with_crawler(mock, crawler):build_from_crawler(mock, crawler, *args, **kwargs)if hasattr(mock, "from_crawler"):mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)- if hasattr(mock, "from_settings"):- assert mock.from_settings.call_count == 0- assert mock.call_count == 0- elif hasattr(mock, "from_settings"):- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)assert mock.call_count == 0else:mock.assert_called_once_with(*args, **kwargs)- # Check usage of correct constructor using three mocks:+ # Check usage of correct constructor using 2 mocks:# 1. with no alternative constructors# 2. with from_crawler() constructor- # 3. with from_settings() and from_crawler() constructorspec_sets = (["__qualname__"],["__qualname__", "from_crawler"],- ["__qualname__", "from_settings", "from_crawler"],)for specs in spec_sets:m = mock.MagicMock(spec_set=specs)- _test_with_crawler(m, settings, crawler)+ _test_with_crawler(m, crawler)m.reset_mock()# Check adoption of crawlerdiff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex cf1c6281d..34fca06c1 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -14,7 +14,6 @@ from scrapy.utils.python import (MutableAsyncChain,MutableChain,binary_is_text,- equal_attributes,get_func_args,memoizemethod_noargs,to_bytes,@@ -150,56 +149,6 @@ def test_binaryistext(value: bytes, expected: bool) -> None:assert binary_is_text(value) is expected[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")-def test_equal_attributes():- class Obj:- pass-- a = Obj()- b = Obj()- # no attributes given return False- assert not equal_attributes(a, b, [])- # nonexistent attributes- assert not equal_attributes(a, b, ["x", "y"])-- a.x = 1- b.x = 1- # equal attribute- assert equal_attributes(a, b, ["x"])-- b.y = 2- # obj1 has no attribute y- assert not equal_attributes(a, b, ["x", "y"])-- a.y = 2- # equal attributes- assert equal_attributes(a, b, ["x", "y"])-- a.y = 1- # different attributes- assert not equal_attributes(a, b, ["x", "y"])-- # test callable- a.meta = {}- b.meta = {}- assert equal_attributes(a, b, ["meta"])-- # compare ['meta']['a']- a.meta["z"] = 1- b.meta["z"] = 1-- get_z = operator.itemgetter("z")- get_meta = operator.attrgetter("meta")-- def compare_z(obj):- return get_z(get_meta(obj))-- assert equal_attributes(a, b, [compare_z, "x"])- # fail z equality- a.meta["z"] = 2- assert not equal_attributes(a, b, [compare_z, "x"])--def test_get_func_args():def f1(a, b, c):passdiff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 9c4cb7159..b25129c87 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -1,32 +1,22 @@from __future__ import annotationsimport json-import warningsfrom hashlib import sha1from weakref import WeakKeyDictionaryimport pytest-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.utils.python import to_bytesfrom scrapy.utils.request import (_fingerprint_cache,fingerprint,- request_authenticate,request_httprepr,request_to_curl,)from scrapy.utils.test import get_crawler[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")-def test_request_authenticate():- r = Request("http://www.example.com")- request_authenticate(r, "someuser", "somepass")- assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="--@pytest.mark.parametrize(("r", "expected"),[@@ -237,25 +227,13 @@ class TestFingerprint:class TestRequestFingerprinter:- def test_default_implementation(self):+ def test_fingerprint(self):crawler = get_crawler()request = Request("https://example.com")assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(request)- def test_deprecated_implementation(self):- settings = {- "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",- }- with warnings.catch_warnings(record=True) as logged_warnings:- crawler = get_crawler(settings_dict=settings)- request = Request("https://example.com")- assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(- request- )- assert logged_warnings-class TestCustomRequestFingerprinter:def test_include_headers(self):@@ -343,57 +321,6 @@ class TestCustomRequestFingerprinter:fingerprint = crawler.request_fingerprinter.fingerprint(request)assert fingerprint == settings["FINGERPRINT"]- def test_from_settings(self):- class RequestFingerprinter:- @classmethod- def from_settings(cls, settings):- return cls(settings)-- def __init__(self, settings):- self._fingerprint = settings["FINGERPRINT"]-- def fingerprint(self, request):- return self._fingerprint-- settings = {- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,- "FINGERPRINT": b"fingerprint",- }- with warnings.catch_warnings():- warnings.simplefilter("ignore", ScrapyDeprecationWarning)- crawler = get_crawler(settings_dict=settings)-- request = Request("http://www.example.com")- fingerprint = crawler.request_fingerprinter.fingerprint(request)- assert fingerprint == settings["FINGERPRINT"]-- def test_from_crawler_and_settings(self):- class RequestFingerprinter:- # This method is ignored due to the presence of from_crawler- @classmethod- def from_settings(cls, settings):- return cls(settings)-- @classmethod- def from_crawler(cls, crawler):- return cls(crawler)-- def __init__(self, crawler):- self._fingerprint = crawler.settings["FINGERPRINT"]-- def fingerprint(self, request):- return self._fingerprint-- settings = {- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,- "FINGERPRINT": b"fingerprint",- }- crawler = get_crawler(settings_dict=settings)-- request = Request("http://www.example.com")- fingerprint = crawler.request_fingerprinter.fingerprint(request)- assert fingerprint == settings["FINGERPRINT"]-class TestRequestToCurl:def _test_request(self, request_object, expected_curl_command):