Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove code deprecated in 2.12.0 (#7126)

Andrey Rakhmatullin61b4befc60cf
diff --git a/docs/conf.py b/docs/conf.py
index 5c5926dbe..9b1db9bb7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -126,7 +126,7 @@ coverage_ignore_pyobjects = [
# The interface methods of duplicate request filtering classes are already
# covered in the interface documentation part of the DUPEFILTER_CLASS
# setting documentation.
- r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",
+ r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$",
# Private exception used by the command-line interface implementation.
r"^scrapy\.exceptions\.UsageError",
# Methods of BaseItemExporter subclasses are only documented in
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5e56de05c..a92070918 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1131,9 +1131,9 @@ interface::
class MyDupeFilter:
@classmethod
- def from_settings(cls, settings):
+ def from_crawler(cls, crawler):
"""Returns an instance of this duplicate request filtering class
- based on the current crawl settings."""
+ based on the current Crawler instance."""
return cls()
def request_seen(self, request):
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 280a0f659..c5b6483e3 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -29,23 +29,6 @@ __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
-def __getattr__(name: str):
- if name == "twisted_version":
- import warnings # noqa: PLC0415 # pylint: disable=reimported
-
- from twisted import version as _txv # noqa: PLC0415
-
- from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
-
- warnings.warn(
- "The scrapy.twisted_version attribute is deprecated, use twisted.version instead",
- ScrapyDeprecationWarning,
- )
- return _txv.major, _txv.minor, _txv.micro
-
- raise AttributeError
-
-
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 76cd376e1..77b17287d 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -182,14 +182,6 @@ class Downloader:
return key
- def _get_slot_key(self, request: Request, spider: Spider | None) -> str:
- warnings.warn(
- "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return self.get_slot_key(request)
-
# passed as download_func into self.middleware.download() in self.fetch()
@inlineCallbacks
def _enqueue_request(
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index d1ba6208a..5f7c9a8c3 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -71,21 +71,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
stacklevel=2,
)
- @classmethod
- def from_settings(
- cls,
- settings: BaseSettings,
- method: int = SSL.SSLv23_METHOD,
- *args: Any,
- **kwargs: Any,
- ) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls._from_settings(settings, method, *args, **kwargs)
-
@classmethod
def from_crawler(
cls,
@@ -94,20 +79,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
*args: Any,
**kwargs: Any,
) -> Self:
- return cls._from_settings(crawler.settings, method, *args, **kwargs)
-
- @classmethod
- def _from_settings(
- cls,
- settings: BaseSettings,
- method: int = SSL.SSLv23_METHOD,
- *args: Any,
- **kwargs: Any,
- ) -> Self:
- tls_verbose_logging: bool = settings.getbool(
+ tls_verbose_logging: bool = crawler.settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
- tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
+ tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
return cls( # type: ignore[misc]
method=method,
tls_verbose_logging=tls_verbose_logging,
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 520ec3380..2e264ade3 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
-import warnings
from pathlib import Path
from typing import TYPE_CHECKING
from warnings import warn
@@ -22,7 +21,6 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import Request
- from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
@@ -30,15 +28,6 @@ class BaseDupeFilter:
"""Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
that does not filter out any request."""
- @classmethod
- def from_settings(cls, settings: BaseSettings) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls()
-
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls()
@@ -88,38 +77,16 @@ class RFPDupeFilter(BaseDupeFilter):
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)
- @classmethod
- def from_settings(
- cls,
- settings: BaseSettings,
- *,
- fingerprinter: RequestFingerprinterProtocol | None = None,
- ) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls._from_settings(settings, fingerprinter=fingerprinter)
-
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.request_fingerprinter
- return cls._from_settings(
- crawler.settings,
+ debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
+ return cls(
+ job_dir(crawler.settings),
+ debug,
fingerprinter=crawler.request_fingerprinter,
)
- @classmethod
- def _from_settings(
- cls,
- settings: BaseSettings,
- *,
- fingerprinter: RequestFingerprinterProtocol | None = None,
- ) -> Self:
- debug = settings.getbool("DUPEFILTER_DEBUG")
- return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
-
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
if fp in self.fingerprints:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a850b330e..adb6fdc33 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -16,7 +16,7 @@ from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
-from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast
+from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast
from urllib.parse import unquote, urlparse
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
@@ -35,8 +35,6 @@ from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
- from collections.abc import Iterable
-
from _typeshed import OpenBinaryMode
from twisted.python.failure import Failure
@@ -54,25 +52,6 @@ UriParamsCallableT: TypeAlias = Callable[
[dict[str, Any], Spider], dict[str, Any] | None
]
-_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
-
-
-def build_storage(
- builder: Callable[..., _StorageT],
- uri: str,
- *args: Any,
- feed_options: dict[str, Any] | None = None,
- preargs: Iterable[Any] = (),
- **kwargs: Any,
-) -> _StorageT:
- warnings.warn(
- "scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- kwargs["feed_options"] = feed_options
- return builder(*preargs, uri, *args, **kwargs)
-
class ItemFilter:
"""
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 88612daa9..84209effa 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -7,7 +7,6 @@ See documentation in docs/topics/email.rst
from __future__ import annotations
import logging
-import warnings
from email import encoders as Encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
@@ -20,7 +19,6 @@ from typing import IO, TYPE_CHECKING, Any
from twisted.internet import ssl
from twisted.internet.defer import Deferred
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
@@ -35,7 +33,6 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
- from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@@ -73,21 +70,9 @@ class MailSender:
self.mailfrom: str = mailfrom
self.debug: bool = debug
- @classmethod
- def from_settings(cls, settings: BaseSettings) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls._from_settings(settings)
-
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- return cls._from_settings(crawler.settings)
-
- @classmethod
- def _from_settings(cls, settings: BaseSettings) -> Self:
+ settings = crawler.settings
return cls(
smtphost=settings["MAIL_HOST"],
mailfrom=settings["MAIL_FROM"],
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 32fae7cc0..21a2ebfa3 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
- from scrapy.settings import BaseSettings, Settings
+ from scrapy.settings import Settings
logger = logging.getLogger(__name__)
@@ -100,43 +100,15 @@ class MiddlewareManager(ABC):
def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:
raise NotImplementedError
- @staticmethod
- def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T:
- if hasattr(objcls, "from_settings"):
- instance = objcls.from_settings(settings) # type: ignore[attr-defined]
- method_name = "from_settings"
- else:
- instance = objcls()
- method_name = "__new__"
- if instance is None:
- raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
- return cast("_T", instance)
-
- @classmethod
- def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls._from_settings(settings, crawler)
-
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- return cls._from_settings(crawler.settings, crawler)
-
- @classmethod
- def _from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
- mwlist = cls._get_mwlist_from_settings(settings)
+ mwlist = cls._get_mwlist_from_settings(crawler.settings)
middlewares = []
enabled = []
for clspath in mwlist:
try:
mwcls = load_object(clspath)
- if crawler is not None:
- mw = build_from_crawler(mwcls, crawler)
- else:
- mw = MiddlewareManager._build_from_settings(mwcls, settings)
+ mw = build_from_crawler(mwcls, crawler)
middlewares.append(mw)
enabled.append(clspath)
except NotConfigured as e:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 38f8537d6..25af0aa72 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -12,7 +12,6 @@ import hashlib
import logging
import mimetypes
import time
-import warnings
from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
@@ -25,17 +24,15 @@ from itemadapter import ItemAdapter
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.threads import deferToThread
-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
-from scrapy.settings import BaseSettings, Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaseInsensitiveDict
-from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import get_func_args, global_object_name, to_bytes
+from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
@@ -49,6 +46,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@@ -161,7 +159,7 @@ class S3FilesStore:
AWS_USE_SSL = None
AWS_VERIFY = None
- POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
+ POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
HEADERS = {
"Cache-Control": "max-age=172800",
}
@@ -280,7 +278,7 @@ class GCSFilesStore:
CACHE_CONTROL = "max-age=172800"
# The bucket's default object ACL will be applied to the object.
- # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
+ # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_crawler().
POLICY = None
def __init__(self, uri: str):
@@ -446,9 +444,8 @@ class FilesPipeline(MediaPipeline):
self,
store_uri: str | PathLike[str],
download_func: Callable[[Request, Spider], Response] | None = None,
- settings: Settings | dict[str, Any] | None = None,
*,
- crawler: Crawler | None = None,
+ crawler: Crawler,
):
if not (store_uri and (store_uri := _to_string(store_uri))):
from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415
@@ -461,18 +458,7 @@ class FilesPipeline(MediaPipeline):
f"to enable {self.__class__.__name__}."
)
- if crawler is not None:
- if settings is not None:
- warnings.warn(
- f"FilesPipeline.__init__() was called with a crawler instance and a settings instance"
- f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
- f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- settings = crawler.settings
- elif isinstance(settings, dict) or settings is None:
- settings = Settings(settings)
+ settings = crawler.settings
cls_name = "FilesPipeline"
self.store: FilesStoreProtocol = self._get_store(store_uri)
resolve = functools.partial(
@@ -490,51 +476,14 @@ class FilesPipeline(MediaPipeline):
resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
)
- super().__init__(
- download_func=download_func,
- settings=settings if not crawler else None,
- crawler=crawler,
- )
-
- @classmethod
- def from_settings(cls, settings: Settings) -> Self:
- warnings.warn(
- f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return cls._from_settings(settings, None)
+ super().__init__(download_func=download_func, crawler=crawler)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- if method_is_overridden(cls, FilesPipeline, "from_settings"):
- warnings.warn(
- f"{global_object_name(cls)} overrides FilesPipeline.from_settings()."
- f" This method is deprecated and won't be called in future Scrapy versions,"
- f" please update your code so that it overrides from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- )
- o = cls.from_settings(crawler.settings)
- o._finish_init(crawler)
- return o
- return cls._from_settings(crawler.settings, crawler)
-
- @classmethod
- def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:
+ settings = crawler.settings
cls._update_stores(settings)
store_uri = settings["FILES_STORE"]
- if "crawler" in get_func_args(cls.__init__):
- o = cls(store_uri, crawler=crawler)
- else:
- o = cls(store_uri, settings=settings)
- if crawler:
- o._finish_init(crawler)
- warnings.warn(
- f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
- " This is deprecated and the argument will be required in future Scrapy versions.",
- category=ScrapyDeprecationWarning,
- )
- return o
+ return cls(store_uri, crawler=crawler)
@classmethod
def _update_stores(cls, settings: BaseSettings) -> None:
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 5ac211dfb..687d29fa5 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -8,19 +8,17 @@ from __future__ import annotations
import functools
import hashlib
-import warnings
from contextlib import suppress
from io import BytesIO
from typing import TYPE_CHECKING, Any
from itemadapter import ItemAdapter
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
-from scrapy.settings import Settings
-from scrapy.utils.python import get_func_args, global_object_name, to_bytes
+from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@@ -58,9 +56,8 @@ class ImagesPipeline(FilesPipeline):
self,
store_uri: str | PathLike[str],
download_func: Callable[[Request, Spider], Response] | None = None,
- settings: Settings | dict[str, Any] | None = None,
*,
- crawler: Crawler | None = None,
+ crawler: Crawler,
):
try:
from PIL import Image, ImageOps # noqa: PLC0415
@@ -72,26 +69,9 @@ class ImagesPipeline(FilesPipeline):
"ImagesPipeline requires installing Pillow 8.3.2 or later"
)
- super().__init__(
- store_uri,
- settings=settings if not crawler else None,
- download_func=download_func,
- crawler=crawler,
- )
-
- if crawler is not None:
- if settings is not None:
- warnings.warn(
- f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance"
- f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
- f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- settings = crawler.settings
- elif isinstance(settings, dict) or settings is None:
- settings = Settings(settings)
+ super().__init__(store_uri, download_func=download_func, crawler=crawler)
+ settings = crawler.settings
resolve = functools.partial(
self._key_for_pipe,
base_class_name="ImagesPipeline",
@@ -121,21 +101,11 @@ class ImagesPipeline(FilesPipeline):
)
@classmethod
- def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ settings = crawler.settings
cls._update_stores(settings)
store_uri = settings["IMAGES_STORE"]
- if "crawler" in get_func_args(cls.__init__):
- o = cls(store_uri, crawler=crawler)
- else:
- o = cls(store_uri, settings=settings)
- if crawler:
- o._finish_init(crawler)
- warnings.warn(
- f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
- " This is deprecated and the argument will be required in future Scrapy versions.",
- category=ScrapyDeprecationWarning,
- )
- return o
+ return cls(store_uri, crawler=crawler)
def file_downloaded(
self,
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index f8fc3687e..438e5e7d6 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -19,14 +19,13 @@ from twisted.python.versions import Version
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.request import NO_CALLBACK, Request
-from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import get_func_args, global_object_name
+from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Callable, Generator
@@ -37,6 +36,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
+ from scrapy.settings import Settings
from scrapy.utils.request import RequestFingerprinterProtocol
@@ -55,10 +55,6 @@ logger = logging.getLogger(__name__)
class MediaPipeline(ABC):
- crawler: Crawler
- _fingerprinter: RequestFingerprinterProtocol
- _modern_init = False
-
LOG_FAILED_RESULTS: bool = True
class SpiderInfo:
@@ -73,24 +69,17 @@ class MediaPipeline(ABC):
def __init__(
self,
download_func: Callable[[Request, Spider], Response] | None = None,
- settings: Settings | dict[str, Any] | None = None,
*,
- crawler: Crawler | None = None,
+ crawler: Crawler,
):
+ self.crawler: Crawler = crawler
+ assert crawler.request_fingerprinter
+ self._fingerprinter: RequestFingerprinterProtocol = (
+ crawler.request_fingerprinter
+ )
self.download_func = download_func
- if crawler is not None:
- if settings is not None:
- warnings.warn(
- f"MediaPipeline.__init__() was called with a crawler instance and a settings instance"
- f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
- f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- settings = crawler.settings
- elif isinstance(settings, dict) or settings is None:
- settings = Settings(settings)
+ settings = crawler.settings
resolve = functools.partial(
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
)
@@ -99,27 +88,6 @@ class MediaPipeline(ABC):
)
self._handle_statuses(self.allow_redirects)
- if crawler:
- self._finish_init(crawler)
- self._modern_init = True
- else:
- warnings.warn(
- f"MediaPipeline.__init__() was called without the crawler argument"
- f" when creating {global_object_name(self.__class__)}."
- f" This is deprecated and the argument will be required in future Scrapy versions.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
- def _finish_init(self, crawler: Crawler) -> None:
- # This was done in from_crawler() before 2.12, now it's done in __init__()
- # if the crawler was passed to it and may be needed to be called in other
- # deprecated code paths explicitly too. After the crawler argument of __init__()
- # becomes mandatory this should be inlined there.
- self.crawler = crawler
- assert crawler.request_fingerprinter
- self._fingerprinter = crawler.request_fingerprinter
-
def _handle_statuses(self, allow_redirects: bool) -> None:
self.handle_httpstatus_list = None
if allow_redirects:
@@ -143,30 +111,7 @@ class MediaPipeline(ABC):
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
- pipe: Self
- if hasattr(cls, "from_settings"):
- pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]
- warnings.warn(
- f"{global_object_name(cls)} has from_settings() and either doesn't have"
- " from_crawler() or calls MediaPipeline.from_crawler() from it,"
- " so from_settings() was used to create the instance of it."
- " This is deprecated and calling from_settings() will be removed"
- " in a future Scrapy version. Please move the initialization code into"
- " from_crawler() or __init__().",
- category=ScrapyDeprecationWarning,
- )
- elif "crawler" in get_func_args(cls.__init__):
- pipe = cls(crawler=crawler)
- else:
- pipe = cls()
- warnings.warn(
- f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
- " This is deprecated and the argument will be required in future Scrapy versions.",
- category=ScrapyDeprecationWarning,
- )
- if not pipe._modern_init:
- pipe._finish_init(crawler)
- return pipe
+ return cls(crawler=crawler)
@_warn_spider_arg
def open_spider(self, spider: Spider | None = None) -> None:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 543e5c043..e66f78c47 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -152,7 +152,6 @@ __all__ = [
"REFERER_ENABLED",
"REFERRER_POLICY",
"REQUEST_FINGERPRINTER_CLASS",
- "REQUEST_FINGERPRINTER_IMPLEMENTATION",
"RETRY_ENABLED",
"RETRY_EXCEPTIONS",
"RETRY_HTTP_CODES",
@@ -451,7 +450,6 @@ REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
-REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
RETRY_ENABLED = True
RETRY_EXCEPTIONS = [
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 39cd6e540..8ca3bc0bd 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -317,31 +317,6 @@ def process_chain(
return d
-def process_chain_both(
- callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
- errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
- input: Any, # noqa: A002
- *a: _P.args,
- **kw: _P.kwargs,
-) -> Deferred:
- """Return a Deferred built by chaining the given callbacks and errbacks"""
- warnings.warn(
- "process_chain_both() is deprecated and will be removed in a future"
- " Scrapy version.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- d: Deferred = Deferred()
- for cb, eb in zip(callbacks, errbacks, strict=False):
- d.addCallback(cb, *a, **kw)
- d.addErrback(eb, *a, **kw)
- if isinstance(input, failure.Failure):
- d.errback(input)
- else:
- d.callback(input)
- return d
-
-
def process_parallel(
callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
input: _T, # noqa: A002
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 1acb06757..7994a2471 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -129,52 +129,10 @@ def rel_has_nofollow(rel: str | None) -> bool:
return rel is not None and "nofollow" in rel.replace(",", " ").split()
-def create_instance(objcls, settings, crawler, *args, **kwargs):
- """Construct a class instance using its ``from_crawler`` or
- ``from_settings`` constructors, if available.
-
- At least one of ``settings`` and ``crawler`` needs to be different from
- ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
- If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
- tried.
-
- ``*args`` and ``**kwargs`` are forwarded to the constructors.
-
- Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
-
- .. versionchanged:: 2.2
- Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
- extension has not been implemented correctly).
- """
- warnings.warn(
- "The create_instance() function is deprecated. "
- "Please use build_from_crawler() instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
- if settings is None:
- if crawler is None:
- raise ValueError("Specify at least one of settings and crawler.")
- settings = crawler.settings
- if crawler and hasattr(objcls, "from_crawler"):
- instance = objcls.from_crawler(crawler, *args, **kwargs)
- method_name = "from_crawler"
- elif hasattr(objcls, "from_settings"):
- instance = objcls.from_settings(settings, *args, **kwargs)
- method_name = "from_settings"
- else:
- instance = objcls(*args, **kwargs)
- method_name = "__new__"
- if instance is None:
- raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
- return instance
-
-
def build_from_crawler(
objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
) -> T:
- """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor.
+ """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor.
.. versionadded:: 2.12
@@ -185,17 +143,6 @@ def build_from_crawler(
if hasattr(objcls, "from_crawler"):
instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]
method_name = "from_crawler"
- elif hasattr(objcls, "from_settings"):
- warnings.warn(
- f"{objcls.__qualname__} has from_settings() but not from_crawler()."
- " This is deprecated and calling from_settings() will be removed in a future"
- " Scrapy version. You can implement a simple from_crawler() that calls"
- " from_settings() with crawler.settings.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined]
- method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d9a5007cc..ca4b4bf28 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,14 +8,12 @@ import gc
import inspect
import re
import sys
-import warnings
import weakref
from collections.abc import AsyncIterator, Iterable, Mapping
from functools import partial, wraps
from itertools import chain
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncgen import as_async_generator
if TYPE_CHECKING:
@@ -32,47 +30,6 @@ _VT = TypeVar("_VT")
_P = ParamSpec("_P")
-def flatten(x: Iterable[Any]) -> list[Any]:
- """flatten(sequence) -> list
-
- Returns a single, flat list which contains all elements retrieved
- from the sequence and all recursively contained sub-sequences
- (iterables).
-
- Examples:
- >>> [1, 2, [3,4], (5,6)]
- [1, 2, [3, 4], (5, 6)]
- >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
- [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
- >>> flatten(["foo", "bar"])
- ['foo', 'bar']
- >>> flatten(["foo", ["baz", 42], "bar"])
- ['foo', 'baz', 42, 'bar']
- """
- warnings.warn(
- "The flatten function is deprecated and will be removed in a future version of Scrapy.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return list(iflatten(x))
-
-
-def iflatten(x: Iterable[Any]) -> Iterable[Any]:
- """iflatten(sequence) -> iterator
-
- Similar to ``.flatten()``, but returns iterator instead"""
- warnings.warn(
- "The iflatten function is deprecated and will be removed in a future version of Scrapy.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- for el in x:
- if is_listlike(el):
- yield from iflatten(el)
- else:
- yield el
-
-
def is_listlike(x: Any) -> bool:
"""
>>> is_listlike("foo")
@@ -289,31 +246,6 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:
return args, kwargs
-def equal_attributes(
- obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None
-) -> bool:
- """Compare two objects attributes"""
- warnings.warn(
- "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- # not attributes given return False by default
- if not attributes:
- return False
-
- temp1, temp2 = object(), object()
- for attr in attributes:
- # support callables like itemgetter
- if callable(attr):
- if attr(obj1) != attr(obj2):
- return False
- elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2):
- return False
- # all attributes equal
- return True
-
-
@overload
def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ...
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 9c1161968..616ab661f 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -7,16 +7,13 @@ from __future__ import annotations
import hashlib
import json
-import warnings
from typing import TYPE_CHECKING, Any, Protocol
from urllib.parse import urlunparse
from weakref import WeakKeyDictionary
-from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy import Request, Spider
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
@@ -120,41 +117,12 @@ class RequestFingerprinter:
return cls(crawler)
def __init__(self, crawler: Crawler | None = None):
- if crawler:
- implementation = crawler.settings.get(
- "REQUEST_FINGERPRINTER_IMPLEMENTATION"
- )
- else:
- implementation = "SENTINEL"
-
- if implementation != "SENTINEL":
- message = (
- "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
- "It will be removed in a future version of Scrapy."
- )
- warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
self._fingerprint = fingerprint
def fingerprint(self, request: Request) -> bytes:
return self._fingerprint(request)
-def request_authenticate(
- request: Request,
- username: str,
- password: str,
-) -> None:
- """Authenticate the given request (in place) using the HTTP basic access
- authentication mechanism (RFC 2617) and the given username and password
- """
- warnings.warn(
- "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- request.headers["Authorization"] = basic_auth_header(username, password)
-
-
def request_httprepr(request: Request) -> bytes:
"""Return the raw HTTP representation (as bytes) of the given request.
This is provided only for reference since it's not the actual stream of
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index bcfae0c00..5d06bbe30 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -1,13 +1,11 @@
import datetime
import decimal
import json
-import warnings
from typing import Any
from itemadapter import ItemAdapter, is_item
from twisted.internet import defer
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
@@ -35,13 +33,3 @@ class ScrapyJSONEncoder(json.JSONEncoder):
if is_item(o):
return ItemAdapter(o).asdict()
return super().default(o)
-
-
-class ScrapyJSONDecoder(json.JSONDecoder):
- def __init__(self, *args, **kwargs):
- warnings.warn(
- "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- super().__init__(*args, **kwargs)
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4c0764396..4e0356350 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -12,7 +12,7 @@ from importlib import import_module
from pathlib import Path
from posixpath import split
from typing import TYPE_CHECKING, Any, TypeVar, cast
-from unittest import TestCase, mock
+from unittest import mock
from twisted.trial.unittest import SkipTest
from twisted.web.client import Agent
@@ -166,20 +166,6 @@ def get_testenv() -> dict[str, str]:
return env
-def assert_samelines(
- testcase: TestCase, text1: str, text2: str, msg: str | None = None
-) -> None:
- """Asserts text1 and text2 have the same lines, ignoring differences in
- line endings between platforms
- """
- warnings.warn(
- "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009
-
-
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
q: asyncio.Queue[_T] = asyncio.Queue()
getter = q.get()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 99d5cbb6d..966cb320e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -756,37 +756,6 @@ class TestBuildFromCrawler:
assert len(w) == 0
assert pipe.store
- def test_has_old_init(self):
- class Pipeline(FilesPipeline):
- def __init__(self, store_uri, download_func=None, settings=None):
- super().__init__(store_uri, download_func, settings)
- self._init_called = True
-
- with warnings.catch_warnings(record=True) as w:
- pipe = Pipeline.from_crawler(self.crawler)
- assert pipe.crawler == self.crawler
- assert pipe._fingerprinter
- assert len(w) == 2
- assert pipe._init_called
-
- def test_has_from_settings(self):
- class Pipeline(FilesPipeline):
- _from_settings_called = False
-
- @classmethod
- def from_settings(cls, settings):
- o = super().from_settings(settings)
- o._from_settings_called = True
- return o
-
- with warnings.catch_warnings(record=True) as w:
- pipe = Pipeline.from_crawler(self.crawler)
- assert pipe.crawler == self.crawler
- assert pipe._fingerprinter
- assert len(w) == 3
- assert pipe.store
- assert pipe._from_settings_called
-
def test_has_from_crawler_and_init(self):
class Pipeline(FilesPipeline):
_from_crawler_called = False
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index cc1d73349..9f174cc5f 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -418,84 +418,6 @@ class TestBuildFromCrawler:
assert pipe._fingerprinter
assert len(w) == 0
- def test_has_old_init(self):
- class Pipeline(UserDefinedPipeline):
- def __init__(self):
- super().__init__()
- self._init_called = True
-
- with warnings.catch_warnings(record=True) as w:
- pipe = Pipeline.from_crawler(self.crawler)
- assert pipe.crawler == self.crawler
- assert pipe._fingerprinter
- assert len(w) == 2
- assert pipe._init_called
-
- def test_has_from_settings(self):
- class Pipeline(UserDefinedPipeline):
- _from_settings_called = False
-
- @classmethod
- def from_settings(cls, settings):
- o = cls()
- o._from_settings_called = True
- return o
-
- with warnings.catch_warnings(record=True) as w:
- pipe = Pipeline.from_crawler(self.crawler)
- assert pipe.crawler == self.crawler
- assert pipe._fingerprinter
- assert len(w) == 2
- assert pipe._from_settings_called
-
- def test_has_from_settings_and_from_crawler(self):
- class Pipeline(UserDefinedPipeline):
- _from_settings_called = False
- _from_crawler_called = False
-
- @classmethod
- def from_settings(cls, settings):
- o = cls()
- o._from_settings_called = True
- return o
-
- @classmethod
- def from_crawler(cls, crawler):
- o = super().from_crawler(crawler)
- o._from_crawler_called = True
- return o
-
- with warnings.catch_warnings(record=True) as w:
- pipe = Pipeline.from_crawler(self.crawler)
- assert pipe.crawler == self.crawler
- assert pipe._fingerprinter
- assert len(w) == 2
- assert pipe._from_settings_called
- assert pipe._from_crawler_called
-
- def test_has_from_settings_and_init(self):
- class Pipeline(UserDefinedPipeline):
- _from_settings_called = False
-
- def __init__(self, store_uri, settings):
- super().__init__()
- self._init_called = True
-
- @classmethod
- def from_settings(cls, settings):
- store_uri = settings["FILES_STORE"]
- o = cls(store_uri, settings=settings)
- o._from_settings_called = True
- return o
-
- with warnings.catch_warnings(record=True) as w:
- pipe = Pipeline.from_crawler(self.crawler)
- assert pipe.crawler == self.crawler
- assert pipe._fingerprinter
- assert len(w) == 2
- assert pipe._from_settings_called
- assert pipe._init_called
-
def test_has_from_crawler_and_init(self):
class Pipeline(UserDefinedPipeline):
_from_crawler_called = False
diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py
index fa6d11123..4c365859b 100644
--- a/tests/test_scrapy__getattr__.py
+++ b/tests/test_scrapy__getattr__.py
@@ -1,20 +1,6 @@
import warnings
-def test_deprecated_twisted_version():
- with warnings.catch_warnings(record=True) as warns:
- from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module
- twisted_version,
- )
-
- assert twisted_version is not None
- assert isinstance(twisted_version, tuple)
- assert (
- "The scrapy.twisted_version attribute is deprecated, use twisted.version instead"
- in warns[0].message.args
- )
-
-
def test_deprecated_concurrent_requests_per_ip_attribute():
with warnings.catch_warnings(record=True) as warns:
from scrapy.settings.default_settings import ( # noqa: PLC0415
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index b330819d9..610fc0ffe 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -9,7 +9,6 @@ from scrapy.item import Field, Item
from scrapy.utils.misc import (
arg_to_iter,
build_from_crawler,
- create_instance,
load_object,
rel_has_nofollow,
set_environ,
@@ -97,98 +96,29 @@ class TestUtilsMisc:
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_create_instance(self):
- settings = mock.MagicMock()
- crawler = mock.MagicMock(spec_set=["settings"])
- args = (True, 100.0)
- kwargs = {"key": "val"}
-
- def _test_with_settings(mock, settings):
- create_instance(mock, settings, None, *args, **kwargs)
- if hasattr(mock, "from_crawler"):
- assert mock.from_crawler.call_count == 0
- if hasattr(mock, "from_settings"):
- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
- assert mock.call_count == 0
- else:
- mock.assert_called_once_with(*args, **kwargs)
-
- def _test_with_crawler(mock, settings, crawler):
- create_instance(mock, settings, crawler, *args, **kwargs)
- if hasattr(mock, "from_crawler"):
- mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
- if hasattr(mock, "from_settings"):
- assert mock.from_settings.call_count == 0
- assert mock.call_count == 0
- elif hasattr(mock, "from_settings"):
- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
- assert mock.call_count == 0
- else:
- mock.assert_called_once_with(*args, **kwargs)
-
- # Check usage of correct constructor using four mocks:
- # 1. with no alternative constructors
- # 2. with from_settings() constructor
- # 3. with from_crawler() constructor
- # 4. with from_settings() and from_crawler() constructor
- spec_sets = (
- ["__qualname__"],
- ["__qualname__", "from_settings"],
- ["__qualname__", "from_crawler"],
- ["__qualname__", "from_settings", "from_crawler"],
- )
- for specs in spec_sets:
- m = mock.MagicMock(spec_set=specs)
- _test_with_settings(m, settings)
- m.reset_mock()
- _test_with_crawler(m, settings, crawler)
-
- # Check adoption of crawler settings
- m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
- create_instance(m, None, crawler, *args, **kwargs)
- m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
-
- with pytest.raises(
- ValueError, match="Specify at least one of settings and crawler"
- ):
- create_instance(m, None, None)
-
- m.from_settings.return_value = None
- with pytest.raises(TypeError):
- create_instance(m, settings, None)
-
def test_build_from_crawler(self):
- settings = mock.MagicMock()
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
- def _test_with_crawler(mock, settings, crawler):
+ def _test_with_crawler(mock, crawler):
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
- if hasattr(mock, "from_settings"):
- assert mock.from_settings.call_count == 0
- assert mock.call_count == 0
- elif hasattr(mock, "from_settings"):
- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
- # Check usage of correct constructor using three mocks:
+ # Check usage of correct constructor using 2 mocks:
# 1. with no alternative constructors
# 2. with from_crawler() constructor
- # 3. with from_settings() and from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_crawler"],
- ["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
- _test_with_crawler(m, settings, crawler)
+ _test_with_crawler(m, crawler)
m.reset_mock()
# Check adoption of crawler
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index cf1c6281d..34fca06c1 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -14,7 +14,6 @@ from scrapy.utils.python import (
MutableAsyncChain,
MutableChain,
binary_is_text,
- equal_attributes,
get_func_args,
memoizemethod_noargs,
to_bytes,
@@ -150,56 +149,6 @@ def test_binaryistext(value: bytes, expected: bool) -> None:
assert binary_is_text(value) is expected
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-def test_equal_attributes():
- class Obj:
- pass
-
- a = Obj()
- b = Obj()
- # no attributes given return False
- assert not equal_attributes(a, b, [])
- # nonexistent attributes
- assert not equal_attributes(a, b, ["x", "y"])
-
- a.x = 1
- b.x = 1
- # equal attribute
- assert equal_attributes(a, b, ["x"])
-
- b.y = 2
- # obj1 has no attribute y
- assert not equal_attributes(a, b, ["x", "y"])
-
- a.y = 2
- # equal attributes
- assert equal_attributes(a, b, ["x", "y"])
-
- a.y = 1
- # different attributes
- assert not equal_attributes(a, b, ["x", "y"])
-
- # test callable
- a.meta = {}
- b.meta = {}
- assert equal_attributes(a, b, ["meta"])
-
- # compare ['meta']['a']
- a.meta["z"] = 1
- b.meta["z"] = 1
-
- get_z = operator.itemgetter("z")
- get_meta = operator.attrgetter("meta")
-
- def compare_z(obj):
- return get_z(get_meta(obj))
-
- assert equal_attributes(a, b, [compare_z, "x"])
- # fail z equality
- a.meta["z"] = 2
- assert not equal_attributes(a, b, [compare_z, "x"])
-
-
def test_get_func_args():
def f1(a, b, c):
pass
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 9c4cb7159..b25129c87 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,32 +1,22 @@
from __future__ import annotations
import json
-import warnings
from hashlib import sha1
from weakref import WeakKeyDictionary
import pytest
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
_fingerprint_cache,
fingerprint,
- request_authenticate,
request_httprepr,
request_to_curl,
)
from scrapy.utils.test import get_crawler
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-def test_request_authenticate():
- r = Request("http://www.example.com")
- request_authenticate(r, "someuser", "somepass")
- assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
-
-
@pytest.mark.parametrize(
("r", "expected"),
[
@@ -237,25 +227,13 @@ class TestFingerprint:
class TestRequestFingerprinter:
- def test_default_implementation(self):
+ def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
- def test_deprecated_implementation(self):
- settings = {
- "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
- }
- with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(settings_dict=settings)
- request = Request("https://example.com")
- assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
- request
- )
- assert logged_warnings
-
class TestCustomRequestFingerprinter:
def test_include_headers(self):
@@ -343,57 +321,6 @@ class TestCustomRequestFingerprinter:
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
- def test_from_settings(self):
- class RequestFingerprinter:
- @classmethod
- def from_settings(cls, settings):
- return cls(settings)
-
- def __init__(self, settings):
- self._fingerprint = settings["FINGERPRINT"]
-
- def fingerprint(self, request):
- return self._fingerprint
-
- settings = {
- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
- "FINGERPRINT": b"fingerprint",
- }
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- crawler = get_crawler(settings_dict=settings)
-
- request = Request("http://www.example.com")
- fingerprint = crawler.request_fingerprinter.fingerprint(request)
- assert fingerprint == settings["FINGERPRINT"]
-
- def test_from_crawler_and_settings(self):
- class RequestFingerprinter:
- # This method is ignored due to the presence of from_crawler
- @classmethod
- def from_settings(cls, settings):
- return cls(settings)
-
- @classmethod
- def from_crawler(cls, crawler):
- return cls(crawler)
-
- def __init__(self, crawler):
- self._fingerprint = crawler.settings["FINGERPRINT"]
-
- def fingerprint(self, request):
- return self._fingerprint
-
- settings = {
- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
- "FINGERPRINT": b"fingerprint",
- }
- crawler = get_crawler(settings_dict=settings)
-
- request = Request("http://www.example.com")
- fingerprint = crawler.request_fingerprinter.fingerprint(request)
- assert fingerprint == settings["FINGERPRINT"]
-
class TestRequestToCurl:
def _test_request(self, request_object, expected_curl_command):