Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecation removals. (#6500)

Andrey Rakhmatullind85c39f5bcd7
diff --git a/conftest.py b/conftest.py
index 2ab3dffd4..77b0e033b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -89,6 +89,30 @@ def requires_uvloop(request):
pytest.skip("uvloop is not installed")
[email protected](autouse=True)
+def requires_botocore(request):
+ if not request.node.get_closest_marker("requires_botocore"):
+ return
+ try:
+ import botocore
+
+ del botocore
+ except ImportError:
+ pytest.skip("botocore is not installed")
+
+
[email protected](autouse=True)
+def requires_boto3(request):
+ if not request.node.get_closest_marker("requires_boto3"):
+ return
+ try:
+ import boto3
+
+ del boto3
+ except ImportError:
+ pytest.skip("boto3 is not installed")
+
+
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 6eb4af9bd..6ffb8ae93 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -278,8 +278,6 @@ Supported options:
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
-* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
-
Usage examples::
$ scrapy crawl myspider
@@ -291,9 +289,6 @@ Usage examples::
$ scrapy crawl -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
- $ scrapy crawl -o myfile -t csv myspider
- [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
-
.. command:: check
check
diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index e2f2dc82b..82eb77cc0 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -41,7 +41,6 @@ _scrapy() {
(runspider)
local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
- {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py'
)
@@ -99,7 +98,6 @@ _scrapy() {
(crawl)
local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
- {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders'
)
diff --git a/pytest.ini b/pytest.ini
index 16983be5e..824c0e9e9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,8 +21,6 @@ markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
requires_uvloop: marks tests as only enabled when uvloop is known to be working
+ requires_botocore: marks tests that need botocore (but not boto3)
+ requires_boto3: marks tests that need botocore and boto3
filterwarnings =
- ignore:scrapy.downloadermiddlewares.decompression is deprecated
- ignore:Module scrapy.utils.reqser is deprecated
- ignore:typing.re is deprecated
- ignore:typing.io is deprecated
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index eccbef040..56199cc01 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -162,12 +162,6 @@ class BaseRunSpiderCommand(ScrapyCommand):
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
)
- parser.add_argument(
- "-t",
- "--output-format",
- metavar="FORMAT",
- help="format to use for dumping items",
- )
def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
super().process_options(args, opts)
@@ -179,8 +173,7 @@ class BaseRunSpiderCommand(ScrapyCommand):
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
- opts.output_format,
- opts.overwrite_output,
+ overwrite_output=opts.overwrite_output,
)
self.settings.set("FEEDS", feeds, priority="cmdline")
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 3e5657d22..de0cf543e 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import pprint
import signal
-import warnings
from typing import TYPE_CHECKING, Any, TypeVar, cast
from twisted.internet.defer import (
@@ -17,7 +16,6 @@ from zope.interface.verify import verifyClass
from scrapy import Spider, signals
from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.logformatter import LogFormatter
@@ -142,10 +140,8 @@ class Crawler:
if self.crawling:
raise RuntimeError("Crawling already taking place")
if self._started:
- warnings.warn(
- "Running Crawler.crawl() more than once is deprecated.",
- ScrapyDeprecationWarning,
- stacklevel=2,
+ raise RuntimeError(
+ "Cannot run Crawler.crawl() more than once on the same instance."
)
self.crawling = self._started = True
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 84678b8e9..a65757972 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import warnings
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, Any
@@ -15,7 +14,6 @@ from scrapy.utils._compression import (
_unbrotli,
_unzstd,
)
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
if TYPE_CHECKING:
@@ -72,21 +70,7 @@ class HttpCompressionMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
- try:
- return cls(crawler=crawler)
- except TypeError:
- warnings.warn(
- "HttpCompressionMiddleware subclasses must either modify "
- "their '__init__' method to support a 'crawler' parameter or "
- "reimplement their 'from_crawler' method.",
- ScrapyDeprecationWarning,
- )
- mw = cls()
- mw.stats = crawler.stats
- mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
- mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
- crawler.signals.connect(mw.open_spider, signals.spider_opened)
- return mw
+ return cls(crawler=crawler)
def open_spider(self, spider: Spider) -> None:
if hasattr(spider, "download_maxsize"):
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 7c0e2280c..9fab172a8 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -12,12 +12,10 @@ once the spider has finished crawling all regular (non-failed) pages.
from __future__ import annotations
-import warnings
from logging import Logger, getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.settings import BaseSettings, Settings
+from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@@ -29,33 +27,13 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.http.request import Request
+ from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
retry_logger = getLogger(__name__)
-def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]:
- if name == "EXCEPTIONS_TO_RETRY":
- warnings.warn(
- "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
- "Use the RETRY_EXCEPTIONS setting instead.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return tuple(
- load_object(x) if isinstance(x, str) else x
- for x in Settings().getlist("RETRY_EXCEPTIONS")
- )
- raise AttributeError(
- f"{self.__class__.__name__!r} object has no attribute {name!r}"
- )
-
-
-class BackwardsCompatibilityMetaclass(type):
- __getattr__ = backwards_compatibility_getattr
-
-
def get_retry_request(
request: Request,
*,
@@ -144,22 +122,17 @@ def get_retry_request(
return None
-class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
+class RetryMiddleware:
def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
-
- try:
- self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")
- except AttributeError:
- # If EXCEPTIONS_TO_RETRY is not "overridden"
- self.exceptions_to_retry = tuple(
- load_object(x) if isinstance(x, str) else x
- for x in settings.getlist("RETRY_EXCEPTIONS")
- )
+ self.exceptions_to_retry = tuple(
+ load_object(x) if isinstance(x, str) else x
+ for x in settings.getlist("RETRY_EXCEPTIONS")
+ )
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@@ -199,5 +172,3 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
)
-
- __getattr__ = backwards_compatibility_getattr
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index eb1698ce5..6ab88dbb4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -26,10 +26,8 @@ from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.settings import Settings
-from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future
-from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler, load_object
@@ -48,13 +46,6 @@ if TYPE_CHECKING:
from scrapy.exporters import BaseItemExporter
from scrapy.settings import BaseSettings
-try:
- import boto3 # noqa: F401
-
- IS_BOTO3_AVAILABLE = True
-except ImportError:
- IS_BOTO3_AVAILABLE = False
-
logger = logging.getLogger(__name__)
@@ -217,8 +208,10 @@ class S3FeedStorage(BlockingFeedStorage):
session_token: str | None = None,
region_name: str | None = None,
):
- if not is_botocore_available():
- raise NotConfigured("missing botocore library")
+ try:
+ import boto3.session
+ except ImportError:
+ raise NotConfigured("missing boto3 library")
u = urlparse(uri)
assert u.hostname
self.bucketname: str = u.hostname
@@ -229,42 +222,16 @@ class S3FeedStorage(BlockingFeedStorage):
self.acl: str | None = acl
self.endpoint_url: str | None = endpoint_url
self.region_name: str | None = region_name
- # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,
- # there seems to be no good way to infer it statically.
- self.s3_client: Any
-
- if IS_BOTO3_AVAILABLE:
- import boto3.session
-
- boto3_session = boto3.session.Session()
-
- self.s3_client = boto3_session.client(
- "s3",
- aws_access_key_id=self.access_key,
- aws_secret_access_key=self.secret_key,
- aws_session_token=self.session_token,
- endpoint_url=self.endpoint_url,
- region_name=self.region_name,
- )
- else:
- warnings.warn(
- "`botocore` usage has been deprecated for S3 feed "
- "export, please use `boto3` to avoid problems",
- category=ScrapyDeprecationWarning,
- )
-
- import botocore.session
-
- botocore_session = botocore.session.get_session()
- self.s3_client = botocore_session.create_client(
- "s3",
- aws_access_key_id=self.access_key,
- aws_secret_access_key=self.secret_key,
- aws_session_token=self.session_token,
- endpoint_url=self.endpoint_url,
- region_name=self.region_name,
- )
+ boto3_session = boto3.session.Session()
+ self.s3_client = boto3_session.client(
+ "s3",
+ aws_access_key_id=self.access_key,
+ aws_secret_access_key=self.secret_key,
+ aws_session_token=self.session_token,
+ endpoint_url=self.endpoint_url,
+ region_name=self.region_name,
+ )
if feed_options and feed_options.get("overwrite", True) is False:
logger.warning(
@@ -295,17 +262,10 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
- kwargs: dict[str, Any]
- if IS_BOTO3_AVAILABLE:
- kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
- self.s3_client.upload_fileobj(
- Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
- )
- else:
- kwargs = {"ACL": self.acl} if self.acl else {}
- self.s3_client.put_object(
- Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
- )
+ kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
+ self.s3_client.upload_fileobj(
+ Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
+ )
file.close()
@@ -464,12 +424,6 @@ class FeedSlot:
self._exporting = False
-_FeedSlot = create_deprecated_class(
- name="_FeedSlot",
- new_class=FeedSlot,
-)
-
-
class FeedExporter:
_pending_deferreds: list[Deferred[None]] = []
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index bbba7d1e1..2c4c9376e 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -8,14 +8,13 @@ from __future__ import annotations
import functools
import hashlib
-import warnings
from contextlib import suppress
from io import BytesIO
from typing import TYPE_CHECKING, Any, cast
from itemadapter import ItemAdapter
-from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import (
@@ -27,7 +26,7 @@ from scrapy.pipelines.files import (
_md5sum,
)
from scrapy.settings import Settings
-from scrapy.utils.python import get_func_args, to_bytes
+from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@@ -42,18 +41,6 @@ if TYPE_CHECKING:
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
-class NoimagesDrop(DropItem):
- """Product with no images exception"""
-
- def __init__(self, *args: Any, **kwargs: Any):
- warnings.warn(
- "The NoimagesDrop class is deprecated",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- super().__init__(*args, **kwargs)
-
-
class ImageException(FileException):
"""General image error exception"""
@@ -120,8 +107,6 @@ class ImagesPipeline(FilesPipeline):
resolve("IMAGES_THUMBS"), self.THUMBS
)
- self._deprecated_convert_image: bool | None = None
-
@classmethod
def from_settings(cls, settings: Settings) -> Self:
s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])
@@ -203,49 +188,25 @@ class ImagesPipeline(FilesPipeline):
f"{self.min_width}x{self.min_height})"
)
- if self._deprecated_convert_image is None:
- self._deprecated_convert_image = "response_body" not in get_func_args(
- self.convert_image
- )
- if self._deprecated_convert_image:
- warnings.warn(
- f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, "
- "overridden method does not accept response_body argument.",
- category=ScrapyDeprecationWarning,
- )
-
- if self._deprecated_convert_image:
- image, buf = self.convert_image(orig_image)
- else:
- image, buf = self.convert_image(
- orig_image, response_body=BytesIO(response.body)
- )
+ image, buf = self.convert_image(
+ orig_image, response_body=BytesIO(response.body)
+ )
yield path, image, buf
for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(
request, thumb_id, response=response, info=info, item=item
)
- if self._deprecated_convert_image:
- thumb_image, thumb_buf = self.convert_image(image, size)
- else:
- thumb_image, thumb_buf = self.convert_image(image, size, buf)
+ thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf)
yield thumb_path, thumb_image, thumb_buf
def convert_image(
self,
image: Image.Image,
size: tuple[int, int] | None = None,
- response_body: BytesIO | None = None,
+ *,
+ response_body: BytesIO,
) -> tuple[Image.Image, BytesIO]:
- if response_body is None:
- warnings.warn(
- f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
- "method called without response_body argument.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
if image.format in ("PNG", "WEBP") and image.mode == "RGBA":
background = self._Image.new("RGBA", image.size, (255, 255, 255))
background.paste(image, image)
@@ -268,7 +229,7 @@ class ImagesPipeline(FilesPipeline):
except AttributeError:
resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
image.thumbnail(size, resampling_filter)
- elif response_body is not None and image.format == "JPEG":
+ elif image.format == "JPEG":
return image, response_body
buf = BytesIO()
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 64cd31c4b..91a49c652 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -3,14 +3,13 @@ from __future__ import annotations
import numbers
import os
import sys
-import warnings
from collections.abc import Iterable
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, cast
-from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
+from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@@ -21,7 +20,7 @@ if TYPE_CHECKING:
def build_component_list(
compdict: MutableMapping[Any, Any],
- custom: Any = None,
+ *,
convert: Callable[[Any], Any] = update_classpath,
) -> list[Any]:
"""Compose a component list from a { class: order } dictionary."""
@@ -60,19 +59,6 @@ def build_component_list(
"please provide a real number or None instead"
)
- if custom is not None:
- warnings.warn(
- "The 'custom' attribute of build_component_list() is deprecated. "
- "Please merge its value into 'compdict' manually or change your "
- "code to use Settings.getwithbase().",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- if isinstance(custom, (list, tuple)):
- _check_components(custom)
- return type(custom)(convert(c) for c in custom) # type: ignore[return-value]
- compdict.update(custom)
-
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
@@ -159,7 +145,7 @@ def feed_complete_default_values_from_settings(
def feed_process_params_from_cli(
settings: BaseSettings,
output: list[str],
- output_format: str | None = None,
+ *,
overwrite_output: list[str] | None = None,
) -> dict[str, dict[str, Any]]:
"""
@@ -186,37 +172,9 @@ def feed_process_params_from_cli(
raise UsageError(
"Please use only one of -o/--output and -O/--overwrite-output"
)
- if output_format:
- raise UsageError(
- "-t/--output-format is a deprecated command line option"
- " and does not work in combination with -O/--overwrite-output."
- " To specify a format please specify it after a colon at the end of the"
- " output URI (i.e. -O <URI>:<FORMAT>)."
- " Example working in the tutorial: "
- "scrapy crawl quotes -O quotes.json:json"
- )
output = overwrite_output
overwrite = True
- if output_format:
- if len(output) == 1:
- check_valid_format(output_format)
- message = (
- "The -t/--output-format command line option is deprecated in favor of "
- "specifying the output format within the output URI using the -o/--output or the"
- " -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation"
- " of the -o or -O option or the following examples for more information. "
- "Examples working in the tutorial: "
- "scrapy crawl quotes -o quotes.csv:csv or "
- "scrapy crawl quotes -O quotes.json:json"
- )
- warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
- return {output[0]: {"format": output_format}}
- raise UsageError(
- "The -t command-line option cannot be used if multiple output "
- "URIs are specified"
- )
-
result: dict[str, dict[str, Any]] = {}
for element in output:
try:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index f8904a9aa..e7bd0b232 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -4,12 +4,11 @@ import asyncio
import sys
from contextlib import suppress
from typing import TYPE_CHECKING, Any, Generic, TypeVar
-from warnings import catch_warnings, filterwarnings, warn
+from warnings import catch_warnings, filterwarnings
from twisted.internet import asyncioreactor, error
from twisted.internet.base import DelayedCall
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
@@ -79,22 +78,6 @@ def set_asyncio_event_loop_policy() -> None:
_get_asyncio_event_loop_policy()
-def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
- warn(
- "Call to deprecated function "
- "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
- "\n"
- "Please use get_event_loop, new_event_loop and set_event_loop"
- " from asyncio instead, as the corresponding policy methods may lead"
- " to unexpected behaviour.\n"
- "This function is replaced by set_asyncio_event_loop_policy and"
- " is meant to be used only when the reactor is being installed.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return _get_asyncio_event_loop_policy()
-
-
def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
policy = asyncio.get_event_loop_policy()
if sys.platform == "win32" and not isinstance(
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 82bdcb0f9..e80cbbb89 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -30,17 +30,9 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
-def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
- for header in headers:
- if header in request.headers:
- yield header
- yield from request.headers.getlist(header)
-
-
_fingerprint_cache: WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]
-]
-_fingerprint_cache = WeakKeyDictionary()
+] = WeakKeyDictionary()
def fingerprint(
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 92a201fd1..37348778c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -8,7 +8,6 @@ import warnings
from pathlib import Path
from typing import Any
-import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from pytest import mark, raises
@@ -82,13 +81,10 @@ class CrawlerTestCase(BaseCrawlerTest):
Crawler(DefaultSpider())
@inlineCallbacks
- def test_crawler_crawl_twice_deprecated(self):
+ def test_crawler_crawl_twice_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
- with pytest.warns(
- ScrapyDeprecationWarning,
- match=r"Running Crawler.crawl\(\) more than once is deprecated",
- ):
+ with raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl()
def test_get_addon(self):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 19cea97ec..6a7597e9f 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -8,6 +8,7 @@ from pathlib import Path
from tempfile import mkdtemp, mkstemp
from unittest import SkipTest, mock
+import pytest
from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal
from twisted.internet import defer, error, reactor
@@ -32,7 +33,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
-from scrapy.utils.test import get_crawler, skip_if_no_boto
+from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import (
Echo,
@@ -824,9 +825,9 @@ class HttpDownloadHandlerMock:
return request
class S3AnonTestCase(unittest.TestCase):
def setUp(self):
- skip_if_no_boto()
crawler = get_crawler()
self.s3reqh = build_from_crawler(
S3DownloadHandler,
@@ -845,6 +846,7 @@ class S3AnonTestCase(unittest.TestCase):
self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/")
class S3TestCase(unittest.TestCase):
download_handler_cls: type = S3DownloadHandler
@@ -856,7 +858,6 @@ class S3TestCase(unittest.TestCase):
AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"
def setUp(self):
- skip_if_no_boto()
crawler = get_crawler()
s3reqh = build_from_crawler(
S3DownloadHandler,
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 7c36f748e..934af6590 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -3,7 +3,6 @@ from io import BytesIO
from logging import WARNING
from pathlib import Path
from unittest import SkipTest, TestCase
-from warnings import catch_warnings
from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding
@@ -12,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
-from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@@ -700,29 +699,3 @@ class HttpCompressionTest(TestCase):
except ImportError:
raise SkipTest("no zstd support (zstandard)")
self._test_download_warnsize_request_meta("zstd")
-
-
-class HttpCompressionSubclassTest(TestCase):
- def test_init_missing_stats(self):
- class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
- def __init__(self):
- super().__init__()
-
- crawler = get_crawler(Spider)
- with catch_warnings(record=True) as caught_warnings:
- HttpCompressionMiddlewareSubclass.from_crawler(crawler)
- messages = tuple(
- str(warning.message)
- for warning in caught_warnings
- if warning.category is ScrapyDeprecationWarning
- )
- self.assertEqual(
- messages,
- (
- (
- "HttpCompressionMiddleware subclasses must either modify "
- "their '__init__' method to support a 'crawler' parameter "
- "or reimplement their 'from_crawler' method."
- ),
- ),
- )
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 661175840..a010865ef 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,6 +1,5 @@
import logging
import unittest
-import warnings
from testfixtures import LogCapture
from twisted.internet import defer
@@ -122,37 +121,6 @@ class RetryTest(unittest.TestCase):
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
- def test_exception_to_retry_custom_middleware(self):
- exc = ValueError
-
- with warnings.catch_warnings(record=True) as warns:
-
- class MyRetryMiddleware(RetryMiddleware):
- EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
-
- self.assertEqual(len(warns), 1)
-
- mw2 = MyRetryMiddleware.from_crawler(self.crawler)
- req = Request(f"http://www.scrapytest.org/{exc.__name__}")
- req = mw2.process_exception(req, exc("foo"), self.spider)
- assert isinstance(req, Request)
- self.assertEqual(req.meta["retry_times"], 1)
-
- def test_exception_to_retry_custom_middleware_self(self):
- class MyRetryMiddleware(RetryMiddleware):
- def process_exception(self, request, exception, spider):
- if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
- return self._retry(request, exception, spider)
-
- exc = OSError
- mw2 = MyRetryMiddleware.from_crawler(self.crawler)
- req = Request(f"http://www.scrapytest.org/{exc.__name__}")
- with warnings.catch_warnings(record=True) as warns:
- req = mw2.process_exception(req, exc("foo"), self.spider)
- assert isinstance(req, Request)
- self.assertEqual(req.meta["retry_times"], 1)
- self.assertEqual(len(warns), 1)
-
def _test_retry_exception(self, req, exception, mw=None):
if mw is None:
mw = self.mw
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f59412ab4..790c347fb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -37,7 +37,6 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter, JsonItemExporter
from scrapy.extensions.feedexport import (
- IS_BOTO3_AVAILABLE,
BlockingFeedStorage,
FeedExporter,
FeedSlot,
@@ -50,7 +49,7 @@ from scrapy.extensions.feedexport import (
)
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
-from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto
+from scrapy.utils.test import get_crawler, mock_google_cloud_storage
from tests.mockserver import MockFTPServer, MockServer
from tests.spiders import ItemSpider
@@ -240,10 +239,8 @@ class BlockingFeedStorageTest(unittest.TestCase):
self.assertRaises(OSError, b.open, spider=spider)
class S3FeedStorageTest(unittest.TestCase):
- def setUp(self):
- skip_if_no_boto()
-
def test_parse_credentials(self):
aws_credentials = {
"AWS_ACCESS_KEY_ID": "settings_key",
@@ -292,38 +289,12 @@ class S3FeedStorageTest(unittest.TestCase):
file = mock.MagicMock()
- if IS_BOTO3_AVAILABLE:
- storage.s3_client = mock.MagicMock()
- yield storage.store(file)
- self.assertEqual(
- storage.s3_client.upload_fileobj.call_args,
- mock.call(Bucket=bucket, Key=key, Fileobj=file),
- )
- else:
- from botocore.stub import Stubber
-
- with Stubber(storage.s3_client) as stub:
- stub.add_response(
- "put_object",
- expected_params={
- "Body": file,
- "Bucket": bucket,
- "Key": key,
- },
- service_response={},
- )
-
- yield storage.store(file)
-
- stub.assert_no_pending_responses()
- self.assertEqual(
- file.method_calls,
- [
- mock.call.seek(0),
- # The call to read does not happen with Stubber
- mock.call.close(),
- ],
- )
+ storage.s3_client = mock.MagicMock()
+ yield storage.store(file)
+ self.assertEqual(
+ storage.s3_client.upload_fileobj.call_args,
+ mock.call(Bucket=bucket, Key=key, Fileobj=file),
+ )
def test_init_without_acl(self):
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@@ -459,14 +430,11 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
- if IS_BOTO3_AVAILABLE:
- acl = (
- storage.s3_client.upload_fileobj.call_args[1]
- .get("ExtraArgs", {})
- .get("ACL")
- )
- else:
- acl = storage.s3_client.put_object.call_args[1].get("ACL")
+ acl = (
+ storage.s3_client.upload_fileobj.call_args[1]
+ .get("ExtraArgs", {})
+ .get("ACL")
+ )
self.assertIsNone(acl)
@defer.inlineCallbacks
@@ -480,10 +448,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
- if IS_BOTO3_AVAILABLE:
- acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
- else:
- acl = storage.s3_client.put_object.call_args[1]["ACL"]
+ acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
self.assertEqual(acl, "custom-acl")
def test_overwrite_default(self):
@@ -2647,9 +2612,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12
)
+ @pytest.mark.requires_boto3
@defer.inlineCallbacks
def test_s3_export(self):
- skip_if_no_boto()
bucket = "mybucket"
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 6ce7fc059..47840caaa 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -11,6 +11,7 @@ from unittest import mock
from urllib.parse import urlparse
import attr
+import pytest
from itemadapter import ItemAdapter
from twisted.internet import defer
from twisted.trial import unittest
@@ -30,7 +31,6 @@ from scrapy.utils.test import (
get_crawler,
get_ftp_content_and_delete,
get_gcs_content_and_delete,
- skip_if_no_boto,
)
from tests.mockserver import MockFTPServer
@@ -507,11 +507,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(fs_store.basedir, str(path))
class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
- skip_if_no_boto()
-
bucket = "mybucket"
key = "export.csv"
uri = f"s3://{bucket}/{key}"
@@ -557,8 +556,6 @@ class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_stat(self):
- skip_if_no_boto()
-
bucket = "mybucket"
key = "export.csv"
uri = f"s3://{bucket}/{key}"
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 2c3b191fe..7561e1fd4 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,24 +1,19 @@
from __future__ import annotations
import dataclasses
-import hashlib
import io
import random
-import warnings
from shutil import rmtree
from tempfile import mkdtemp
-from unittest.mock import patch
import attr
from itemadapter import ItemAdapter
from twisted.trial import unittest
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.item import Field, Item
-from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop
+from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
skip_pillow: str | None
try:
@@ -159,7 +154,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
with self.assertRaises(ImageException):
next(self.pipeline.get_images(response=resp3, request=req, info=object()))
- def test_get_images_new(self):
+ def test_get_images(self):
self.pipeline.min_width = 0
self.pipeline.min_height = 0
self.pipeline.thumbs = {"small": (20, 20)}
@@ -185,101 +180,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
self.assertEqual(thumb_img, thumb_img)
self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())
- def test_get_images_old(self):
- self.pipeline.thumbs = {"small": (20, 20)}
- orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
- resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
- req = Request(url="https://dev.mydeco.com/mydeco.gif")
-
- def overridden_convert_image(image, size=None):
- im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
- return im, buf
-
- with patch.object(self.pipeline, "convert_image", overridden_convert_image):
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- get_images_gen = self.pipeline.get_images(
- response=resp, request=req, info=object()
- )
- path, new_im, new_buf = next(get_images_gen)
- self.assertEqual(
- path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
- )
- self.assertEqual(orig_im.mode, new_im.mode)
- self.assertEqual(orig_im.getcolors(), new_im.getcolors())
- self.assertEqual(buf.getvalue(), new_buf.getvalue())
-
- thumb_path, thumb_img, thumb_buf = next(get_images_gen)
- self.assertEqual(
- thumb_path,
- "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
- )
- self.assertEqual(orig_im.mode, thumb_img.mode)
- self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
- self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
-
- expected_warning_msg = (
- ".convert_image() method overridden in a deprecated way, "
- "overridden method does not accept response_body argument."
- )
- self.assertEqual(
- len(
- [
- warning
- for warning in w
- if expected_warning_msg in str(warning.message)
- ]
- ),
- 1,
- )
-
- def test_convert_image_old(self):
- # tests for old API
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- SIZE = (100, 100)
- # straight forward case: RGB and JPEG
- COLOUR = (0, 127, 255)
- im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)
- converted, _ = self.pipeline.convert_image(im)
- self.assertEqual(converted.mode, "RGB")
- self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
-
- # check that thumbnail keep image ratio
- thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))
- self.assertEqual(thumbnail.mode, "RGB")
- self.assertEqual(thumbnail.size, (10, 10))
-
- # transparency case: RGBA and PNG
- COLOUR = (0, 127, 255, 50)
- im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
- converted, _ = self.pipeline.convert_image(im)
- self.assertEqual(converted.mode, "RGB")
- self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
-
- # transparency case with palette: P and PNG
- COLOUR = (0, 127, 255, 50)
- im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
- im = im.convert("P")
- converted, _ = self.pipeline.convert_image(im)
- self.assertEqual(converted.mode, "RGB")
- self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
-
- # ensure that we received deprecation warnings
- expected_warning_msg = ".convert_image() method called in a deprecated way"
- self.assertTrue(
- len(
- [
- warning
- for warning in w
- if expected_warning_msg in str(warning.message)
- ]
- )
- == 4
- )
-
- def test_convert_image_new(self):
- # tests for new API
+ def test_convert_image(self):
SIZE = (100, 100)
# straight forward case: RGB and JPEG
COLOUR = (0, 127, 255)
@@ -313,19 +214,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
-class DeprecatedImagesPipeline(ImagesPipeline):
- def file_key(self, url):
- return self.image_key(url)
-
- def image_key(self, url):
- image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
- return f"empty/{image_guid}.jpg"
-
- def thumb_key(self, url, thumb_id):
- thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
- return f"thumbsup/{thumb_id}/{thumb_guid}.jpg"
-
-
class ImagesPipelineTestCaseFieldsMixin:
skip = skip_pillow
@@ -627,23 +515,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value)
-class NoimagesDropTestCase(unittest.TestCase):
- def test_deprecation_warning(self):
- arg = ""
- with warnings.catch_warnings(record=True) as w:
- NoimagesDrop(arg)
- self.assertEqual(len(w), 1)
- self.assertEqual(w[0].category, ScrapyDeprecationWarning)
- with warnings.catch_warnings(record=True) as w:
-
- class SubclassedNoimagesDrop(NoimagesDrop):
- pass
-
- SubclassedNoimagesDrop(arg)
- self.assertEqual(len(w), 1)
- self.assertEqual(w[0].category, ScrapyDeprecationWarning)
-
-
def _create_image(format, *a, **kw):
buf = io.BytesIO()
Image.new(*a, **kw).save(buf, format)
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index dc3f01d57..2ce7948eb 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,9 +1,6 @@
import unittest
-import warnings
-import pytest
-
-from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
+from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
arglist_to_dict,
@@ -20,50 +17,6 @@ class BuildComponentListTest(unittest.TestCase):
build_component_list(d, convert=lambda x: x), ["one", "four", "three"]
)
- def test_backward_compatible_build_dict(self):
- base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}
- custom = {"two": None, "three": 8, "four": 4}
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- self.assertEqual(
- build_component_list(base, custom, convert=lambda x: x),
- ["one", "four", "five", "three"],
- )
-
- def test_return_list(self):
- custom = ["a", "b", "c"]
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- self.assertEqual(
- build_component_list(None, custom, convert=lambda x: x), custom
- )
-
- def test_map_dict(self):
- custom = {"one": 1, "two": 2, "three": 3}
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- self.assertEqual(
- build_component_list({}, custom, convert=lambda x: x.upper()),
- ["ONE", "TWO", "THREE"],
- )
-
- def test_map_list(self):
- custom = ["a", "b", "c"]
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- self.assertEqual(
- build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]
- )
-
- def test_duplicate_components_in_dict(self):
- duplicate_dict = {"one": 1, "two": 2, "ONE": 4}
- with self.assertRaises(ValueError):
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- build_component_list({}, duplicate_dict, convert=lambda x: x.lower())
-
- def test_duplicate_components_in_list(self):
- duplicate_list = ["a", "b", "a"]
- with self.assertRaises(ValueError) as cm:
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- build_component_list(None, duplicate_list, convert=lambda x: x)
- self.assertIn(str(duplicate_list), str(cm.exception))
-
def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence
duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)
@@ -92,11 +45,6 @@ class BuildComponentListTest(unittest.TestCase):
"c": 22222222222222222222,
}
self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])
- # raise exception for invalid values
- d = {"one": "5"}
- with self.assertRaises(ValueError):
- with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
- build_component_list({}, d, convert=lambda x: x)
class UtilsConfTestCase(unittest.TestCase):
@@ -115,7 +63,6 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli,
settings,
["items.dat"],
- "noformat",
)
def test_feed_export_config_mismatch(self):
@@ -125,18 +72,8 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli,
settings,
["items1.dat", "items2.dat"],
- "noformat",
)
- def test_feed_export_config_backward_compatible(self):
- with warnings.catch_warnings(record=True) as cw:
- settings = Settings()
- self.assertEqual(
- {"items.dat": {"format": "csv"}},
- feed_process_params_from_cli(settings, ["items.dat"], "csv"),
- )
- self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
-
def test_feed_export_config_explicit_formats(self):
settings = Settings()
self.assertEqual(
@@ -174,7 +111,9 @@ class FeedExportConfigTestCase(unittest.TestCase):
settings = Settings()
self.assertEqual(
{"output.json": {"format": "json", "overwrite": True}},
- feed_process_params_from_cli(settings, [], None, ["output.json"]),
+ feed_process_params_from_cli(
+ settings, [], overwrite_output=["output.json"]
+ ),
)
def test_output_and_overwrite_output(self):
@@ -182,8 +121,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli(
Settings(),
["output1.json"],
- None,
- ["output2.json"],
+ overwrite_output=["output2.json"],
)
def test_feed_complete_default_values_from_settings_empty(self):
diff --git a/tox.ini b/tox.ini
index a526fc120..5783a0e61 100644
--- a/tox.ini
+++ b/tox.ini
@@ -241,7 +241,7 @@ deps =
{[testenv]deps}
botocore>=1.4.87
commands =
- pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}
[testenv:botocore-pinned]
basepython = {[pinned]basepython}
@@ -252,4 +252,4 @@ install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands =
- pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}