Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecation removals. (#6500)
Andrey Rakhmatullind85c39f5bcd7
diff --git a/conftest.py b/conftest.pyindex 2ab3dffd4..77b0e033b 100644--- a/conftest.py+++ b/conftest.py@@ -89,6 +89,30 @@ def requires_uvloop(request):pytest.skip("uvloop is not installed")[email protected](autouse=True)+def requires_botocore(request):+ if not request.node.get_closest_marker("requires_botocore"):+ return+ try:+ import botocore++ del botocore+ except ImportError:+ pytest.skip("botocore is not installed")++[email protected](autouse=True)+def requires_boto3(request):+ if not request.node.get_closest_marker("requires_boto3"):+ return+ try:+ import boto3++ del boto3+ except ImportError:+ pytest.skip("boto3 is not installed")++def pytest_configure(config):if config.getoption("--reactor") == "asyncio":install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")diff --git a/docs/topics/commands.rst b/docs/topics/commands.rstindex 6eb4af9bd..6ffb8ae93 100644--- a/docs/topics/commands.rst+++ b/docs/topics/commands.rst@@ -278,8 +278,6 @@ Supported options:* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)-* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``-Usage examples::$ scrapy crawl myspider@@ -291,9 +289,6 @@ Usage examples::$ scrapy crawl -O myfile:json myspider[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]- $ scrapy crawl -o myfile -t csv myspider- [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]-.. command:: checkcheckdiff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completionindex e2f2dc82b..82eb77cc0 100644--- a/extras/scrapy_zsh_completion+++ b/extras/scrapy_zsh_completion@@ -41,7 +41,6 @@ _scrapy() {(runspider)local options=({'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'- {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)''*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)''1:spider file:_files -g \*.py')@@ -99,7 +98,6 @@ _scrapy() {(crawl)local options=({'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'- {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)''*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)''1:spider:_scrapy_spiders')diff --git a/pytest.ini b/pytest.iniindex 16983be5e..824c0e9e9 100644--- a/pytest.ini+++ b/pytest.ini@@ -21,8 +21,6 @@ markers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passedonly_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passedrequires_uvloop: marks tests as only enabled when uvloop is known to be working+ requires_botocore: marks tests that need botocore (but not boto3)+ requires_boto3: marks tests that need botocore and boto3filterwarnings =- ignore:scrapy.downloadermiddlewares.decompression is deprecated- ignore:Module scrapy.utils.reqser is deprecated- ignore:typing.re is deprecated- ignore:typing.io is deprecateddiff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex eccbef040..56199cc01 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -162,12 +162,6 @@ class BaseRunSpiderCommand(ScrapyCommand):help="dump scraped items into FILE, overwriting any existing file,"" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",)- parser.add_argument(- "-t",- "--output-format",- metavar="FORMAT",- help="format to use for dumping items",- )def process_options(self, args: list[str], opts: argparse.Namespace) -> None:super().process_options(args, opts)@@ -179,8 +173,7 @@ class BaseRunSpiderCommand(ScrapyCommand):feeds = feed_process_params_from_cli(self.settings,opts.output,- opts.output_format,- opts.overwrite_output,+ overwrite_output=opts.overwrite_output,)self.settings.set("FEEDS", feeds, priority="cmdline")diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 3e5657d22..de0cf543e 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport loggingimport pprintimport signal-import warningsfrom typing import TYPE_CHECKING, Any, TypeVar, castfrom twisted.internet.defer import (@@ -17,7 +16,6 @@ from zope.interface.verify import verifyClassfrom scrapy import Spider, signalsfrom scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEngine-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extension import ExtensionManagerfrom scrapy.interfaces import ISpiderLoaderfrom scrapy.logformatter import LogFormatter@@ -142,10 +140,8 @@ class Crawler:if self.crawling:raise RuntimeError("Crawling already taking place")if self._started:- warnings.warn(- "Running Crawler.crawl() more than once is deprecated.",- ScrapyDeprecationWarning,- stacklevel=2,+ raise RuntimeError(+ "Cannot run Crawler.crawl() more than once on the same instance.")self.crawling = self._started = Truediff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 84678b8e9..a65757972 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,6 +1,5 @@from __future__ import annotations-import warningsfrom itertools import chainfrom logging import getLoggerfrom typing import TYPE_CHECKING, Any@@ -15,7 +14,6 @@ from scrapy.utils._compression import (_unbrotli,_unzstd,)-from scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.gz import gunzipif TYPE_CHECKING:@@ -72,21 +70,7 @@ class HttpCompressionMiddleware:def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("COMPRESSION_ENABLED"):raise NotConfigured- try:- return cls(crawler=crawler)- except TypeError:- warnings.warn(- "HttpCompressionMiddleware subclasses must either modify "- "their '__init__' method to support a 'crawler' parameter or "- "reimplement their 'from_crawler' method.",- ScrapyDeprecationWarning,- )- mw = cls()- mw.stats = crawler.stats- mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")- mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")- crawler.signals.connect(mw.open_spider, signals.spider_opened)- return mw+ return cls(crawler=crawler)def open_spider(self, spider: Spider) -> None:if hasattr(spider, "download_maxsize"):diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 7c0e2280c..9fab172a8 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -12,12 +12,10 @@ once the spider has finished crawling all regular (non-failed) pages.from __future__ import annotations-import warningsfrom logging import Logger, getLogger-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning-from scrapy.settings import BaseSettings, Settings+from scrapy.exceptions import NotConfiguredfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import global_object_namefrom scrapy.utils.response import response_status_message@@ -29,33 +27,13 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.http import Responsefrom scrapy.http.request import Request+ from scrapy.settings import BaseSettingsfrom scrapy.spiders import Spiderretry_logger = getLogger(__name__)-def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]:- if name == "EXCEPTIONS_TO_RETRY":- warnings.warn(- "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "- "Use the RETRY_EXCEPTIONS setting instead.",- ScrapyDeprecationWarning,- stacklevel=2,- )- return tuple(- load_object(x) if isinstance(x, str) else x- for x in Settings().getlist("RETRY_EXCEPTIONS")- )- raise AttributeError(- f"{self.__class__.__name__!r} object has no attribute {name!r}"- )---class BackwardsCompatibilityMetaclass(type):- __getattr__ = backwards_compatibility_getattr--def get_retry_request(request: Request,*,@@ -144,22 +122,17 @@ def get_retry_request(return None-class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):+class RetryMiddleware:def __init__(self, settings: BaseSettings):if not settings.getbool("RETRY_ENABLED"):raise NotConfiguredself.max_retry_times = settings.getint("RETRY_TIMES")self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")-- try:- self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")- except AttributeError:- # If EXCEPTIONS_TO_RETRY is not "overridden"- self.exceptions_to_retry = tuple(- load_object(x) if isinstance(x, str) else x- for x in settings.getlist("RETRY_EXCEPTIONS")- )+ self.exceptions_to_retry = tuple(+ load_object(x) if isinstance(x, str) else x+ for x in settings.getlist("RETRY_EXCEPTIONS")+ )@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:@@ -199,5 +172,3 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):max_retry_times=max_retry_times,priority_adjust=priority_adjust,)-- __getattr__ = backwards_compatibility_getattrdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex eb1698ce5..6ab88dbb4 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -26,10 +26,8 @@ from scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.extensions.postprocessing import PostProcessingManagerfrom scrapy.settings import Settings-from scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.defer import maybe_deferred_to_future-from scrapy.utils.deprecate import create_deprecated_classfrom scrapy.utils.ftp import ftp_store_filefrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import build_from_crawler, load_object@@ -48,13 +46,6 @@ if TYPE_CHECKING:from scrapy.exporters import BaseItemExporterfrom scrapy.settings import BaseSettings-try:- import boto3 # noqa: F401-- IS_BOTO3_AVAILABLE = True-except ImportError:- IS_BOTO3_AVAILABLE = False-logger = logging.getLogger(__name__)@@ -217,8 +208,10 @@ class S3FeedStorage(BlockingFeedStorage):session_token: str | None = None,region_name: str | None = None,):- if not is_botocore_available():- raise NotConfigured("missing botocore library")+ try:+ import boto3.session+ except ImportError:+ raise NotConfigured("missing boto3 library")u = urlparse(uri)assert u.hostnameself.bucketname: str = u.hostname@@ -229,42 +222,16 @@ class S3FeedStorage(BlockingFeedStorage):self.acl: str | None = aclself.endpoint_url: str | None = endpoint_urlself.region_name: str | None = region_name- # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,- # there seems to be no good way to infer it statically.- self.s3_client: Any-- if IS_BOTO3_AVAILABLE:- import boto3.session-- boto3_session = boto3.session.Session()-- self.s3_client = boto3_session.client(- "s3",- aws_access_key_id=self.access_key,- aws_secret_access_key=self.secret_key,- aws_session_token=self.session_token,- endpoint_url=self.endpoint_url,- region_name=self.region_name,- )- else:- warnings.warn(- "`botocore` usage has been deprecated for S3 feed "- "export, please use `boto3` to avoid problems",- category=ScrapyDeprecationWarning,- )-- import botocore.session-- botocore_session = botocore.session.get_session()- self.s3_client = botocore_session.create_client(- "s3",- aws_access_key_id=self.access_key,- aws_secret_access_key=self.secret_key,- aws_session_token=self.session_token,- endpoint_url=self.endpoint_url,- region_name=self.region_name,- )+ boto3_session = boto3.session.Session()+ self.s3_client = boto3_session.client(+ "s3",+ aws_access_key_id=self.access_key,+ aws_secret_access_key=self.secret_key,+ aws_session_token=self.session_token,+ endpoint_url=self.endpoint_url,+ region_name=self.region_name,+ )if feed_options and feed_options.get("overwrite", True) is False:logger.warning(@@ -295,17 +262,10 @@ class S3FeedStorage(BlockingFeedStorage):def _store_in_thread(self, file: IO[bytes]) -> None:file.seek(0)- kwargs: dict[str, Any]- if IS_BOTO3_AVAILABLE:- kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}- self.s3_client.upload_fileobj(- Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs- )- else:- kwargs = {"ACL": self.acl} if self.acl else {}- self.s3_client.put_object(- Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs- )+ kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}+ self.s3_client.upload_fileobj(+ Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs+ )file.close()@@ -464,12 +424,6 @@ class FeedSlot:self._exporting = False-_FeedSlot = create_deprecated_class(- name="_FeedSlot",- new_class=FeedSlot,-)--class FeedExporter:_pending_deferreds: list[Deferred[None]] = []diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex bbba7d1e1..2c4c9376e 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -8,14 +8,13 @@ from __future__ import annotationsimport functoolsimport hashlib-import warningsfrom contextlib import suppressfrom io import BytesIOfrom typing import TYPE_CHECKING, Any, castfrom itemadapter import ItemAdapter-from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import (@@ -27,7 +26,7 @@ from scrapy.pipelines.files import (_md5sum,)from scrapy.settings import Settings-from scrapy.utils.python import get_func_args, to_bytes+from scrapy.utils.python import to_bytesif TYPE_CHECKING:from collections.abc import Callable, Iterable@@ -42,18 +41,6 @@ if TYPE_CHECKING:from scrapy.pipelines.media import FileInfoOrError, MediaPipeline-class NoimagesDrop(DropItem):- """Product with no images exception"""-- def __init__(self, *args: Any, **kwargs: Any):- warnings.warn(- "The NoimagesDrop class is deprecated",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- super().__init__(*args, **kwargs)--class ImageException(FileException):"""General image error exception"""@@ -120,8 +107,6 @@ class ImagesPipeline(FilesPipeline):resolve("IMAGES_THUMBS"), self.THUMBS)- self._deprecated_convert_image: bool | None = None-@classmethoddef from_settings(cls, settings: Settings) -> Self:s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])@@ -203,49 +188,25 @@ class ImagesPipeline(FilesPipeline):f"{self.min_width}x{self.min_height})")- if self._deprecated_convert_image is None:- self._deprecated_convert_image = "response_body" not in get_func_args(- self.convert_image- )- if self._deprecated_convert_image:- warnings.warn(- f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, "- "overridden method does not accept response_body argument.",- category=ScrapyDeprecationWarning,- )-- if self._deprecated_convert_image:- image, buf = self.convert_image(orig_image)- else:- image, buf = self.convert_image(- orig_image, response_body=BytesIO(response.body)- )+ image, buf = self.convert_image(+ orig_image, response_body=BytesIO(response.body)+ )yield path, image, buffor thumb_id, size in self.thumbs.items():thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)- if self._deprecated_convert_image:- thumb_image, thumb_buf = self.convert_image(image, size)- else:- thumb_image, thumb_buf = self.convert_image(image, size, buf)+ thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf)yield thumb_path, thumb_image, thumb_bufdef convert_image(self,image: Image.Image,size: tuple[int, int] | None = None,- response_body: BytesIO | None = None,+ *,+ response_body: BytesIO,) -> tuple[Image.Image, BytesIO]:- if response_body is None:- warnings.warn(- f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "- "method called without response_body argument.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-if image.format in ("PNG", "WEBP") and image.mode == "RGBA":background = self._Image.new("RGBA", image.size, (255, 255, 255))background.paste(image, image)@@ -268,7 +229,7 @@ class ImagesPipeline(FilesPipeline):except AttributeError:resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]image.thumbnail(size, resampling_filter)- elif response_body is not None and image.format == "JPEG":+ elif image.format == "JPEG":return image, response_bodybuf = BytesIO()diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 64cd31c4b..91a49c652 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -3,14 +3,13 @@ from __future__ import annotationsimport numbersimport osimport sys-import warningsfrom collections.abc import Iterablefrom configparser import ConfigParserfrom operator import itemgetterfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Callable, cast-from scrapy.exceptions import ScrapyDeprecationWarning, UsageError+from scrapy.exceptions import UsageErrorfrom scrapy.settings import BaseSettingsfrom scrapy.utils.deprecate import update_classpathfrom scrapy.utils.python import without_none_values@@ -21,7 +20,7 @@ if TYPE_CHECKING:def build_component_list(compdict: MutableMapping[Any, Any],- custom: Any = None,+ *,convert: Callable[[Any], Any] = update_classpath,) -> list[Any]:"""Compose a component list from a { class: order } dictionary."""@@ -60,19 +59,6 @@ def build_component_list("please provide a real number or None instead")- if custom is not None:- warnings.warn(- "The 'custom' attribute of build_component_list() is deprecated. "- "Please merge its value into 'compdict' manually or change your "- "code to use Settings.getwithbase().",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- if isinstance(custom, (list, tuple)):- _check_components(custom)- return type(custom)(convert(c) for c in custom) # type: ignore[return-value]- compdict.update(custom)-_validate_values(compdict)compdict = without_none_values(_map_keys(compdict))return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]@@ -159,7 +145,7 @@ def feed_complete_default_values_from_settings(def feed_process_params_from_cli(settings: BaseSettings,output: list[str],- output_format: str | None = None,+ *,overwrite_output: list[str] | None = None,) -> dict[str, dict[str, Any]]:"""@@ -186,37 +172,9 @@ def feed_process_params_from_cli(raise UsageError("Please use only one of -o/--output and -O/--overwrite-output")- if output_format:- raise UsageError(- "-t/--output-format is a deprecated command line option"- " and does not work in combination with -O/--overwrite-output."- " To specify a format please specify it after a colon at the end of the"- " output URI (i.e. -O <URI>:<FORMAT>)."- " Example working in the tutorial: "- "scrapy crawl quotes -O quotes.json:json"- )output = overwrite_outputoverwrite = True- if output_format:- if len(output) == 1:- check_valid_format(output_format)- message = (- "The -t/--output-format command line option is deprecated in favor of "- "specifying the output format within the output URI using the -o/--output or the"- " -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation"- " of the -o or -O option or the following examples for more information. "- "Examples working in the tutorial: "- "scrapy crawl quotes -o quotes.csv:csv or "- "scrapy crawl quotes -O quotes.json:json"- )- warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)- return {output[0]: {"format": output_format}}- raise UsageError(- "The -t command-line option cannot be used if multiple output "- "URIs are specified"- )-result: dict[str, dict[str, Any]] = {}for element in output:try:diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex f8904a9aa..e7bd0b232 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -4,12 +4,11 @@ import asyncioimport sysfrom contextlib import suppressfrom typing import TYPE_CHECKING, Any, Generic, TypeVar-from warnings import catch_warnings, filterwarnings, warn+from warnings import catch_warnings, filterwarningsfrom twisted.internet import asyncioreactor, errorfrom twisted.internet.base import DelayedCall-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import load_objectif TYPE_CHECKING:@@ -79,22 +78,6 @@ def set_asyncio_event_loop_policy() -> None:_get_asyncio_event_loop_policy()-def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:- warn(- "Call to deprecated function "- "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"- "\n"- "Please use get_event_loop, new_event_loop and set_event_loop"- " from asyncio instead, as the corresponding policy methods may lead"- " to unexpected behaviour.\n"- "This function is replaced by set_asyncio_event_loop_policy and"- " is meant to be used only when the reactor is being installed.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- return _get_asyncio_event_loop_policy()--def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:policy = asyncio.get_event_loop_policy()if sys.platform == "win32" and not isinstance(diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 82bdcb0f9..e80cbbb89 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -30,17 +30,9 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler-def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:- for header in headers:- if header in request.headers:- yield header- yield from request.headers.getlist(header)--_fingerprint_cache: WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]-]-_fingerprint_cache = WeakKeyDictionary()+] = WeakKeyDictionary()def fingerprint(diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 92a201fd1..37348778c 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -8,7 +8,6 @@ import warningsfrom pathlib import Pathfrom typing import Any-import pytestfrom packaging.version import parse as parse_versionfrom pexpect.popen_spawn import PopenSpawnfrom pytest import mark, raises@@ -82,13 +81,10 @@ class CrawlerTestCase(BaseCrawlerTest):Crawler(DefaultSpider())@inlineCallbacks- def test_crawler_crawl_twice_deprecated(self):+ def test_crawler_crawl_twice_unsupported(self):crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)yield crawler.crawl()- with pytest.warns(- ScrapyDeprecationWarning,- match=r"Running Crawler.crawl\(\) more than once is deprecated",- ):+ with raises(RuntimeError, match="more than once on the same instance"):yield crawler.crawl()def test_get_addon(self):diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 19cea97ec..6a7597e9f 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -8,6 +8,7 @@ from pathlib import Pathfrom tempfile import mkdtemp, mkstempfrom unittest import SkipTest, mock+import pytestfrom testfixtures import LogCapturefrom twisted.cred import checkers, credentials, portalfrom twisted.internet import defer, error, reactor@@ -32,7 +33,7 @@ from scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytes-from scrapy.utils.test import get_crawler, skip_if_no_boto+from scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver import (Echo,@@ -824,9 +825,9 @@ class HttpDownloadHandlerMock:return request[email protected]_botocoreclass S3AnonTestCase(unittest.TestCase):def setUp(self):- skip_if_no_boto()crawler = get_crawler()self.s3reqh = build_from_crawler(S3DownloadHandler,@@ -845,6 +846,7 @@ class S3AnonTestCase(unittest.TestCase):self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/")[email protected]_botocoreclass S3TestCase(unittest.TestCase):download_handler_cls: type = S3DownloadHandler@@ -856,7 +858,6 @@ class S3TestCase(unittest.TestCase):AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"def setUp(self):- skip_if_no_boto()crawler = get_crawler()s3reqh = build_from_crawler(S3DownloadHandler,diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 7c36f748e..934af6590 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -3,7 +3,6 @@ from io import BytesIOfrom logging import WARNINGfrom pathlib import Pathfrom unittest import SkipTest, TestCase-from warnings import catch_warningsfrom testfixtures import LogCapturefrom w3lib.encoding import resolve_encoding@@ -12,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import (ACCEPTED_ENCODINGS,HttpCompressionMiddleware,)-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning+from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider@@ -700,29 +699,3 @@ class HttpCompressionTest(TestCase):except ImportError:raise SkipTest("no zstd support (zstandard)")self._test_download_warnsize_request_meta("zstd")---class HttpCompressionSubclassTest(TestCase):- def test_init_missing_stats(self):- class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):- def __init__(self):- super().__init__()-- crawler = get_crawler(Spider)- with catch_warnings(record=True) as caught_warnings:- HttpCompressionMiddlewareSubclass.from_crawler(crawler)- messages = tuple(- str(warning.message)- for warning in caught_warnings- if warning.category is ScrapyDeprecationWarning- )- self.assertEqual(- messages,- (- (- "HttpCompressionMiddleware subclasses must either modify "- "their '__init__' method to support a 'crawler' parameter "- "or reimplement their 'from_crawler' method."- ),- ),- )diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 661175840..a010865ef 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -1,6 +1,5 @@import loggingimport unittest-import warningsfrom testfixtures import LogCapturefrom twisted.internet import defer@@ -122,37 +121,6 @@ class RetryTest(unittest.TestCase):req = Request(f"http://www.scrapytest.org/{exc.__name__}")self._test_retry_exception(req, exc("foo"), mw)- def test_exception_to_retry_custom_middleware(self):- exc = ValueError-- with warnings.catch_warnings(record=True) as warns:-- class MyRetryMiddleware(RetryMiddleware):- EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)-- self.assertEqual(len(warns), 1)-- mw2 = MyRetryMiddleware.from_crawler(self.crawler)- req = Request(f"http://www.scrapytest.org/{exc.__name__}")- req = mw2.process_exception(req, exc("foo"), self.spider)- assert isinstance(req, Request)- self.assertEqual(req.meta["retry_times"], 1)-- def test_exception_to_retry_custom_middleware_self(self):- class MyRetryMiddleware(RetryMiddleware):- def process_exception(self, request, exception, spider):- if isinstance(exception, self.EXCEPTIONS_TO_RETRY):- return self._retry(request, exception, spider)-- exc = OSError- mw2 = MyRetryMiddleware.from_crawler(self.crawler)- req = Request(f"http://www.scrapytest.org/{exc.__name__}")- with warnings.catch_warnings(record=True) as warns:- req = mw2.process_exception(req, exc("foo"), self.spider)- assert isinstance(req, Request)- self.assertEqual(req.meta["retry_times"], 1)- self.assertEqual(len(warns), 1)-def _test_retry_exception(self, req, exception, mw=None):if mw is None:mw = self.mwdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex f59412ab4..790c347fb 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -37,7 +37,6 @@ from scrapy import signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.exporters import CsvItemExporter, JsonItemExporterfrom scrapy.extensions.feedexport import (- IS_BOTO3_AVAILABLE,BlockingFeedStorage,FeedExporter,FeedSlot,@@ -50,7 +49,7 @@ from scrapy.extensions.feedexport import ()from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode-from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto+from scrapy.utils.test import get_crawler, mock_google_cloud_storagefrom tests.mockserver import MockFTPServer, MockServerfrom tests.spiders import ItemSpider@@ -240,10 +239,8 @@ class BlockingFeedStorageTest(unittest.TestCase):self.assertRaises(OSError, b.open, spider=spider)[email protected]_boto3class S3FeedStorageTest(unittest.TestCase):- def setUp(self):- skip_if_no_boto()-def test_parse_credentials(self):aws_credentials = {"AWS_ACCESS_KEY_ID": "settings_key",@@ -292,38 +289,12 @@ class S3FeedStorageTest(unittest.TestCase):file = mock.MagicMock()- if IS_BOTO3_AVAILABLE:- storage.s3_client = mock.MagicMock()- yield storage.store(file)- self.assertEqual(- storage.s3_client.upload_fileobj.call_args,- mock.call(Bucket=bucket, Key=key, Fileobj=file),- )- else:- from botocore.stub import Stubber-- with Stubber(storage.s3_client) as stub:- stub.add_response(- "put_object",- expected_params={- "Body": file,- "Bucket": bucket,- "Key": key,- },- service_response={},- )-- yield storage.store(file)-- stub.assert_no_pending_responses()- self.assertEqual(- file.method_calls,- [- mock.call.seek(0),- # The call to read does not happen with Stubber- mock.call.close(),- ],- )+ storage.s3_client = mock.MagicMock()+ yield storage.store(file)+ self.assertEqual(+ storage.s3_client.upload_fileobj.call_args,+ mock.call(Bucket=bucket, Key=key, Fileobj=file),+ )def test_init_without_acl(self):storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")@@ -459,14 +430,11 @@ class S3FeedStorageTest(unittest.TestCase):storage.s3_client = mock.MagicMock()yield storage.store(BytesIO(b"test file"))- if IS_BOTO3_AVAILABLE:- acl = (- storage.s3_client.upload_fileobj.call_args[1]- .get("ExtraArgs", {})- .get("ACL")- )- else:- acl = storage.s3_client.put_object.call_args[1].get("ACL")+ acl = (+ storage.s3_client.upload_fileobj.call_args[1]+ .get("ExtraArgs", {})+ .get("ACL")+ )self.assertIsNone(acl)@defer.inlineCallbacks@@ -480,10 +448,7 @@ class S3FeedStorageTest(unittest.TestCase):storage.s3_client = mock.MagicMock()yield storage.store(BytesIO(b"test file"))- if IS_BOTO3_AVAILABLE:- acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]- else:- acl = storage.s3_client.put_object.call_args[1]["ACL"]+ acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]self.assertEqual(acl, "custom-acl")def test_overwrite_default(self):@@ -2647,9 +2612,9 @@ class BatchDeliveriesTest(FeedExportTestBase):crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12)+ @pytest.mark.requires_boto3@defer.inlineCallbacksdef test_s3_export(self):- skip_if_no_boto()bucket = "mybucket"items = [self.MyItem({"foo": "bar1", "egg": "spam1"}),diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 6ce7fc059..47840caaa 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -11,6 +11,7 @@ from unittest import mockfrom urllib.parse import urlparseimport attr+import pytestfrom itemadapter import ItemAdapterfrom twisted.internet import deferfrom twisted.trial import unittest@@ -30,7 +31,6 @@ from scrapy.utils.test import (get_crawler,get_ftp_content_and_delete,get_gcs_content_and_delete,- skip_if_no_boto,)from tests.mockserver import MockFTPServer@@ -507,11 +507,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):self.assertEqual(fs_store.basedir, str(path))[email protected]_botocoreclass TestS3FilesStore(unittest.TestCase):@defer.inlineCallbacksdef test_persist(self):- skip_if_no_boto()-bucket = "mybucket"key = "export.csv"uri = f"s3://{bucket}/{key}"@@ -557,8 +556,6 @@ class TestS3FilesStore(unittest.TestCase):@defer.inlineCallbacksdef test_stat(self):- skip_if_no_boto()-bucket = "mybucket"key = "export.csv"uri = f"s3://{bucket}/{key}"diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 2c3b191fe..7561e1fd4 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -1,24 +1,19 @@from __future__ import annotationsimport dataclasses-import hashlibimport ioimport random-import warningsfrom shutil import rmtreefrom tempfile import mkdtemp-from unittest.mock import patchimport attrfrom itemadapter import ItemAdapterfrom twisted.trial import unittest-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Item-from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop+from scrapy.pipelines.images import ImageException, ImagesPipelinefrom scrapy.settings import Settings-from scrapy.utils.python import to_bytesskip_pillow: str | Nonetry:@@ -159,7 +154,7 @@ class ImagesPipelineTestCase(unittest.TestCase):with self.assertRaises(ImageException):next(self.pipeline.get_images(response=resp3, request=req, info=object()))- def test_get_images_new(self):+ def test_get_images(self):self.pipeline.min_width = 0self.pipeline.min_height = 0self.pipeline.thumbs = {"small": (20, 20)}@@ -185,101 +180,7 @@ class ImagesPipelineTestCase(unittest.TestCase):self.assertEqual(thumb_img, thumb_img)self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())- def test_get_images_old(self):- self.pipeline.thumbs = {"small": (20, 20)}- orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))- resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())- req = Request(url="https://dev.mydeco.com/mydeco.gif")-- def overridden_convert_image(image, size=None):- im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))- return im, buf-- with patch.object(self.pipeline, "convert_image", overridden_convert_image):- with warnings.catch_warnings(record=True) as w:- warnings.simplefilter("always")- get_images_gen = self.pipeline.get_images(- response=resp, request=req, info=object()- )- path, new_im, new_buf = next(get_images_gen)- self.assertEqual(- path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"- )- self.assertEqual(orig_im.mode, new_im.mode)- self.assertEqual(orig_im.getcolors(), new_im.getcolors())- self.assertEqual(buf.getvalue(), new_buf.getvalue())-- thumb_path, thumb_img, thumb_buf = next(get_images_gen)- self.assertEqual(- thumb_path,- "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",- )- self.assertEqual(orig_im.mode, thumb_img.mode)- self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())- self.assertEqual(buf.getvalue(), thumb_buf.getvalue())-- expected_warning_msg = (- ".convert_image() method overridden in a deprecated way, "- "overridden method does not accept response_body argument."- )- self.assertEqual(- len(- [- warning- for warning in w- if expected_warning_msg in str(warning.message)- ]- ),- 1,- )-- def test_convert_image_old(self):- # tests for old API- with warnings.catch_warnings(record=True) as w:- warnings.simplefilter("always")- SIZE = (100, 100)- # straight forward case: RGB and JPEG- COLOUR = (0, 127, 255)- im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)- converted, _ = self.pipeline.convert_image(im)- self.assertEqual(converted.mode, "RGB")- self.assertEqual(converted.getcolors(), [(10000, COLOUR)])-- # check that thumbnail keep image ratio- thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))- self.assertEqual(thumbnail.mode, "RGB")- self.assertEqual(thumbnail.size, (10, 10))-- # transparency case: RGBA and PNG- COLOUR = (0, 127, 255, 50)- im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)- converted, _ = self.pipeline.convert_image(im)- self.assertEqual(converted.mode, "RGB")- self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])-- # transparency case with palette: P and PNG- COLOUR = (0, 127, 255, 50)- im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)- im = im.convert("P")- converted, _ = self.pipeline.convert_image(im)- self.assertEqual(converted.mode, "RGB")- self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])-- # ensure that we received deprecation warnings- expected_warning_msg = ".convert_image() method called in a deprecated way"- self.assertTrue(- len(- [- warning- for warning in w- if expected_warning_msg in str(warning.message)- ]- )- == 4- )-- def test_convert_image_new(self):- # tests for new API+ def test_convert_image(self):SIZE = (100, 100)# straight forward case: RGB and JPEGCOLOUR = (0, 127, 255)@@ -313,19 +214,6 @@ class ImagesPipelineTestCase(unittest.TestCase):self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])-class DeprecatedImagesPipeline(ImagesPipeline):- def file_key(self, url):- return self.image_key(url)-- def image_key(self, url):- image_guid = hashlib.sha1(to_bytes(url)).hexdigest()- return f"empty/{image_guid}.jpg"-- def thumb_key(self, url, thumb_id):- thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()- return f"thumbsup/{thumb_id}/{thumb_guid}.jpg"--class ImagesPipelineTestCaseFieldsMixin:skip = skip_pillow@@ -627,23 +515,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value)-class NoimagesDropTestCase(unittest.TestCase):- def test_deprecation_warning(self):- arg = ""- with warnings.catch_warnings(record=True) as w:- NoimagesDrop(arg)- self.assertEqual(len(w), 1)- self.assertEqual(w[0].category, ScrapyDeprecationWarning)- with warnings.catch_warnings(record=True) as w:-- class SubclassedNoimagesDrop(NoimagesDrop):- pass-- SubclassedNoimagesDrop(arg)- self.assertEqual(len(w), 1)- self.assertEqual(w[0].category, ScrapyDeprecationWarning)--def _create_image(format, *a, **kw):buf = io.BytesIO()Image.new(*a, **kw).save(buf, format)diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex dc3f01d57..2ce7948eb 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -1,9 +1,6 @@import unittest-import warnings-import pytest--from scrapy.exceptions import ScrapyDeprecationWarning, UsageError+from scrapy.exceptions import UsageErrorfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.conf import (arglist_to_dict,@@ -20,50 +17,6 @@ class BuildComponentListTest(unittest.TestCase):build_component_list(d, convert=lambda x: x), ["one", "four", "three"])- def test_backward_compatible_build_dict(self):- base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}- custom = {"two": None, "three": 8, "four": 4}- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- self.assertEqual(- build_component_list(base, custom, convert=lambda x: x),- ["one", "four", "five", "three"],- )-- def test_return_list(self):- custom = ["a", "b", "c"]- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- self.assertEqual(- build_component_list(None, custom, convert=lambda x: x), custom- )-- def test_map_dict(self):- custom = {"one": 1, "two": 2, "three": 3}- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- self.assertEqual(- build_component_list({}, custom, convert=lambda x: x.upper()),- ["ONE", "TWO", "THREE"],- )-- def test_map_list(self):- custom = ["a", "b", "c"]- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- self.assertEqual(- build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]- )-- def test_duplicate_components_in_dict(self):- duplicate_dict = {"one": 1, "two": 2, "ONE": 4}- with self.assertRaises(ValueError):- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- build_component_list({}, duplicate_dict, convert=lambda x: x.lower())-- def test_duplicate_components_in_list(self):- duplicate_list = ["a", "b", "a"]- with self.assertRaises(ValueError) as cm:- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- build_component_list(None, duplicate_list, convert=lambda x: x)- self.assertIn(str(duplicate_list), str(cm.exception))-def test_duplicate_components_in_basesettings(self):# Higher priority takes precedenceduplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)@@ -92,11 +45,6 @@ class BuildComponentListTest(unittest.TestCase):"c": 22222222222222222222,}self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])- # raise exception for invalid values- d = {"one": "5"}- with self.assertRaises(ValueError):- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):- build_component_list({}, d, convert=lambda x: x)class UtilsConfTestCase(unittest.TestCase):@@ -115,7 +63,6 @@ class FeedExportConfigTestCase(unittest.TestCase):feed_process_params_from_cli,settings,["items.dat"],- "noformat",)def test_feed_export_config_mismatch(self):@@ -125,18 +72,8 @@ class FeedExportConfigTestCase(unittest.TestCase):feed_process_params_from_cli,settings,["items1.dat", "items2.dat"],- "noformat",)- def test_feed_export_config_backward_compatible(self):- with warnings.catch_warnings(record=True) as cw:- settings = Settings()- self.assertEqual(- {"items.dat": {"format": "csv"}},- feed_process_params_from_cli(settings, ["items.dat"], "csv"),- )- self.assertEqual(cw[0].category, ScrapyDeprecationWarning)-def test_feed_export_config_explicit_formats(self):settings = Settings()self.assertEqual(@@ -174,7 +111,9 @@ class FeedExportConfigTestCase(unittest.TestCase):settings = Settings()self.assertEqual({"output.json": {"format": "json", "overwrite": True}},- feed_process_params_from_cli(settings, [], None, ["output.json"]),+ feed_process_params_from_cli(+ settings, [], overwrite_output=["output.json"]+ ),)def test_output_and_overwrite_output(self):@@ -182,8 +121,7 @@ class FeedExportConfigTestCase(unittest.TestCase):feed_process_params_from_cli(Settings(),["output1.json"],- None,- ["output2.json"],+ overwrite_output=["output2.json"],)def test_feed_complete_default_values_from_settings_empty(self):diff --git a/tox.ini b/tox.iniindex a526fc120..5783a0e61 100644--- a/tox.ini+++ b/tox.ini@@ -241,7 +241,7 @@ deps ={[testenv]deps}botocore>=1.4.87commands =- pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}[testenv:botocore-pinned]basepython = {[pinned]basepython}@@ -252,4 +252,4 @@ install_command = {[pinned]install_command}setenv ={[pinned]setenv}commands =- pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}