Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate spider args of middlewares and pipelines. (#7006)
Andrey Rakhmatullined63fa94d698
diff --git a/docs/faq.rst b/docs/faq.rstindex 1d09a0e63..fdf0582de 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -349,7 +349,7 @@ method for this purpose. For example:class MultiplyItemsMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):for item_or_request in result:if isinstance(item_or_request, Request):continuediff --git a/docs/news.rst b/docs/news.rstindex 92320ac6a..6a019e2fb 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -44,6 +44,20 @@ Backward-incompatible changes- :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`+- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for+ handling ``open_spider()`` and ``close_spider()`` component methods. As+ this code was only used for pipelines it was moved into+ :class:`scrapy.pipelines.ItemPipelineManager`. This change should only+ affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`.+ The following code was moved:++ - ``scrapy.middleware.MiddlewareManager.open_spider()``++ - ``scrapy.middleware.MiddlewareManager.close_spider()``++ - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that+ processes ``open_spider()`` and ``close_spider()`` component methods.+- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request`now returns a coroutine, previously it returned a:class:`~twisted.internet.defer.Deferred` object or ``None``. Thediff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rstindex b9d780528..fdd3dcd87 100644--- a/docs/topics/coroutines.rst+++ b/docs/topics/coroutines.rst@@ -191,7 +191,7 @@ shorter and cleaner:adapter["field"] = datareturn item- def process_item(self, item, spider):+ def process_item(self, item):adapter = ItemAdapter(item)dfd = db.get_some_data(adapter["id"])dfd.addCallback(self._update_item, item)@@ -205,7 +205,7 @@ becomes:class DbPipeline:- async def process_item(self, item, spider):+ async def process_item(self, item):adapter = ItemAdapter(item)adapter["field"] = await db.get_some_data(adapter["id"])return item@@ -421,12 +421,12 @@ For example:.. code-block:: pythonclass UniversalSpiderMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):for r in result:# ... do something with ryield r- async def process_spider_output_async(self, response, result, spider):+ async def process_spider_output_async(self, response, result):async for r in result:# ... do something with ryield rdiff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex 2fbe3f75d..d3186ef5c 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -67,7 +67,7 @@ value of one of their fields:self.year_to_exporter[year] = (exporter, xml_file)return self.year_to_exporter[year][0]- def process_item(self, item, spider):+ def process_item(self, item):exporter = self._exporter_for_item(item)exporter.export_item(item)return itemdiff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rstindex 13a5b88f7..e67cf06c8 100644--- a/docs/topics/item-pipeline.rst+++ b/docs/topics/item-pipeline.rst@@ -26,7 +26,7 @@ Writing your own item pipelineEach item pipeline is a :ref:`component <topics-components>` that mustimplement the following method:-.. method:: process_item(self, item, spider)+.. method:: process_item(self, item)This method is called for every item pipeline component.@@ -42,25 +42,16 @@ implement the following method::param item: the scraped item:type item: :ref:`item object <item-types>`- :param spider: the spider which scraped the item- :type spider: :class:`~scrapy.Spider` object-Additionally, they may also implement the following methods:-.. method:: open_spider(self, spider)+.. method:: open_spider(self)This method is called when the spider is opened.- :param spider: the spider which was opened- :type spider: :class:`~scrapy.Spider` object--.. method:: close_spider(self, spider)+.. method:: close_spider(self)This method is called when the spider is closed.- :param spider: the spider which was closed- :type spider: :class:`~scrapy.Spider` object-Item pipeline example=====================@@ -82,7 +73,7 @@ contain a price:class PricePipeline:vat_factor = 1.15- def process_item(self, item, spider):+ def process_item(self, item):adapter = ItemAdapter(item)if adapter.get("price"):if adapter.get("price_excludes_vat"):@@ -107,13 +98,13 @@ format:class JsonWriterPipeline:- def open_spider(self, spider):+ def open_spider(self):self.file = open("items.jsonl", "w")- def close_spider(self, spider):+ def close_spider(self):self.file.close()- def process_item(self, item, spider):+ def process_item(self, item):line = json.dumps(ItemAdapter(item).asdict()) + "\n"self.file.write(line)return item@@ -153,14 +144,14 @@ The main point of this example is to show how to :ref:`get the crawlermongo_db=crawler.settings.get("MONGO_DATABASE", "items"),)- def open_spider(self, spider):+ def open_spider(self):self.client = pymongo.MongoClient(self.mongo_uri)self.db = self.client[self.mongo_db]- def close_spider(self, spider):+ def close_spider(self):self.client.close()- def process_item(self, item, spider):+ def process_item(self, item):self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())return item@@ -198,12 +189,19 @@ item.SPLASH_URL = "http://localhost:8050/render.png?url={}"- async def process_item(self, item, spider):+ def __init__(crawler):+ self.crawler = crawler++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)++ async def process_item(self, item):adapter = ItemAdapter(item)encoded_item_url = quote(adapter["url"])screenshot_url = self.SPLASH_URL.format(encoded_item_url)request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)- response = await spider.crawler.engine.download_async(request)+ response = await self.crawler.engine.download_async(request)if response.status != 200:# Error happened, return item.@@ -238,7 +236,7 @@ returns multiples items with the same id:def __init__(self):self.ids_seen = set()- def process_item(self, item, spider):+ def process_item(self, item):adapter = ItemAdapter(item)if adapter["id"] in self.ids_seen:raise DropItem(f"Item ID already seen: {adapter['id']}")diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 4e9519eef..9dc26d703 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -554,7 +554,7 @@ When writing an item pipeline, you can force a different log level by settingclass MyPipeline:- def process_item(self, item, spider):+ def process_item(self, item):if not item.get("price"):raise DropItem("Missing price data", log_level="INFO")return itemdiff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 2eb59fe44..0f81e409e 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -94,7 +94,7 @@ one or more of these methods:def process_start_requests(self, start, spider):yield from start- .. method:: process_spider_input(response, spider)+ .. method:: process_spider_input(response)This method is called for each response that goes through the spidermiddleware and into the spider, for processing.@@ -116,11 +116,7 @@ one or more of these methods::param response: the response being processed:type response: :class:`~scrapy.http.Response` object- :param spider: the spider for which this response is intended- :type spider: :class:`~scrapy.Spider` object--- .. method:: process_spider_output(response, result, spider)+ .. method:: process_spider_output(response, result)This method is called with the results returned from the Spider, afterit has processed the response.@@ -149,10 +145,7 @@ one or more of these methods::type result: an iterable of :class:`~scrapy.Request` objects and:ref:`item objects <topics-items>`- :param spider: the spider whose result is being processed- :type spider: :class:`~scrapy.Spider` object-- .. method:: process_spider_output_async(response, result, spider)+ .. method:: process_spider_output_async(response, result):async:.. versionadded:: 2.7@@ -161,7 +154,7 @@ one or more of these methods:which will be called instead of :meth:`process_spider_output` if``result`` is an :term:`asynchronous iterable`.- .. method:: process_spider_exception(response, exception, spider)+ .. method:: process_spider_exception(response, exception)This method is called when a spider or :meth:`process_spider_output`method (from a previous spider middleware) raises an exception.@@ -186,8 +179,6 @@ one or more of these methods::param exception: the exception raised:type exception: :exc:`Exception` object- :param spider: the spider which raised the exception- :type spider: :class:`~scrapy.Spider` objectBase class for custom spider middlewares----------------------------------------diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 9f92e906d..75246710d 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -117,9 +117,10 @@ class SpiderMiddlewareManager(MiddlewareManager):)def _add_middleware(self, mw: Any) -> None:- super()._add_middleware(mw)if hasattr(mw, "process_spider_input"):self.methods["process_spider_input"].append(mw.process_spider_input)+ self._check_mw_method_spider_arg(mw.process_spider_input)+if self._use_start_requests:if hasattr(mw, "process_start_requests"):self.methods["process_start_requests"].appendleft(@@ -127,10 +128,19 @@ class SpiderMiddlewareManager(MiddlewareManager):)elif hasattr(mw, "process_start"):self.methods["process_start"].appendleft(mw.process_start)+process_spider_output = self._get_async_method_pair(mw, "process_spider_output")self.methods["process_spider_output"].appendleft(process_spider_output)+ if callable(process_spider_output):+ self._check_mw_method_spider_arg(process_spider_output)+ elif isinstance(process_spider_output, tuple):+ for m in process_spider_output:+ self._check_mw_method_spider_arg(m)+process_spider_exception = getattr(mw, "process_spider_exception", None)self.methods["process_spider_exception"].appendleft(process_spider_exception)+ if process_spider_exception is not None:+ self._check_mw_method_spider_arg(process_spider_exception)async def _process_spider_input(self,@@ -141,7 +151,10 @@ class SpiderMiddlewareManager(MiddlewareManager):for method in self.methods["process_spider_input"]:method = cast("Callable", method)try:- result = method(response=response, spider=self._spider)+ if method in self._mw_methods_requiring_spider:+ result = method(response=response, spider=self._spider)+ else:+ result = method(response=response)if result is not None:msg = (f"{global_object_name(method)} must return None "@@ -212,7 +225,12 @@ class SpiderMiddlewareManager(MiddlewareManager):if method is None:continuemethod = cast("Callable", method)- result = method(response=response, exception=exception, spider=self._spider)+ if method in self._mw_methods_requiring_spider:+ result = method(+ response=response, exception=exception, spider=self._spider+ )+ else:+ result = method(response=response, exception=exception)if _isiterable(result):# stop exception handling by handing control over to the# process_spider_output chain if an iterable has been returned@@ -298,7 +316,12 @@ class SpiderMiddlewareManager(MiddlewareManager):)recovered = MutableChain(recovered_collected)# might fail directly if the output value is not a generator- result = method(response=response, result=result, spider=self._spider)+ if method in self._mw_methods_requiring_spider:+ result = method(+ response=response, result=result, spider=self._spider+ )+ else:+ result = method(response=response, result=result)except Exception as ex:exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (self._process_spider_exception(response, ex, method_index + 1)@@ -421,7 +444,7 @@ class SpiderMiddlewareManager(MiddlewareManager):if self._use_start_requests:sync_start = iter(self._spider.start_requests())sync_start = await self._process_chain(- "process_start_requests", sync_start, self._spider+ "process_start_requests", sync_start, always_add_spider=True)start: AsyncIterator[Any] = as_async_generator(sync_start)else:diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 8d2ade120..be41b52e4 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -8,7 +8,8 @@ from collections import defaultdict, dequefrom typing import TYPE_CHECKING, Any, TypeVar, castfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning-from scrapy.utils.defer import _process_chain, process_parallel+from scrapy.utils.defer import ensure_awaitable+from scrapy.utils.deprecate import argument_is_requiredfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import global_object_name@@ -31,7 +32,6 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)_T = TypeVar("_T")-_T2 = TypeVar("_T2")class MiddlewareManager(ABC):@@ -56,6 +56,7 @@ class MiddlewareManager(ABC):self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = (defaultdict(deque))+ self._mw_methods_requiring_spider: set[Callable] = set()for mw in middlewares:self._add_middleware(mw)@@ -158,34 +159,53 @@ class MiddlewareManager(ABC):)return cls(*middlewares, crawler=crawler)- def _add_middleware(self, mw: Any) -> None:- if hasattr(mw, "open_spider"):- self.methods["open_spider"].append(mw.open_spider)- if hasattr(mw, "close_spider"):- self.methods["close_spider"].appendleft(mw.close_spider)+ def _add_middleware(self, mw: Any) -> None: # noqa: B027+ pass- def _process_parallel(- self, methodname: str, obj: _T, *args: Any- ) -> Deferred[list[_T2]]:+ def _check_mw_method_spider_arg(self, method: Callable) -> None:+ if argument_is_required(method, "spider"):+ warnings.warn(+ f"{method.__qualname__}() requires a spider argument,"+ f" this is deprecated and the argument will not be passed in future Scrapy versions."+ f" If you need to access the spider instance you can save the crawler instance"+ f" passed to from_crawler() and use its spider attribute.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ self._mw_methods_requiring_spider.add(method)++ async def _process_chain(+ self,+ methodname: str,+ obj: _T,+ *args: Any,+ add_spider: bool = False,+ always_add_spider: bool = False,+ ) -> _T:methods = cast(- "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname]+ "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]+ )+ for method in methods:+ if always_add_spider or (+ add_spider and method in self._mw_methods_requiring_spider+ ):+ obj = await ensure_awaitable(method(obj, *(*args, self._spider)))+ else:+ obj = await ensure_awaitable(method(obj, *args))+ return obj++ def open_spider(+ self, spider: Spider | None = None+ ) -> Deferred[list[None]]: # pragma: no cover+ raise NotImplementedError(+ "MiddlewareManager.open_spider() is no longer implemented"+ " and will be removed in a future Scrapy version.")- return process_parallel(methods, obj, *args)- async def _process_chain(self, methodname: str, obj: _T, *args: Any) -> _T:- methods = cast(- "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]+ def close_spider(+ self, spider: Spider | None = None+ ) -> Deferred[list[None]]: # pragma: no cover+ raise NotImplementedError(+ "MiddlewareManager.close_spider() is no longer implemented"+ " and will be removed in a future Scrapy version.")- return await _process_chain(methods, obj, *args)-- def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:- if spider:- self._warn_spider_arg("open_spider")- self._set_compat_spider(spider)- return self._process_parallel("open_spider", self._spider)-- def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:- if spider:- self._warn_spider_arg("close_spider")- self._set_compat_spider(spider)- return self._process_parallel("close_spider", self._spider)diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 8fe40ab13..b7ec928e8 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -7,16 +7,20 @@ See documentation in docs/item-pipeline.rstfrom __future__ import annotationsimport warnings-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Callable, cast++from twisted.internet.defer import Deferred, DeferredListfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_list-from scrapy.utils.defer import deferred_from_coro+from scrapy.utils.defer import deferred_from_coro, maybeDeferred_corofrom scrapy.utils.python import global_object_nameif TYPE_CHECKING:- from twisted.internet.defer import Deferred+ from collections.abc import Iterable++ from twisted.python.failure import Failurefrom scrapy import Spiderfrom scrapy.settings import Settings@@ -30,9 +34,15 @@ class ItemPipelineManager(MiddlewareManager):return build_component_list(settings.getwithbase("ITEM_PIPELINES"))def _add_middleware(self, pipe: Any) -> None:- super()._add_middleware(pipe)+ if hasattr(pipe, "open_spider"):+ self.methods["open_spider"].append(pipe.open_spider)+ self._check_mw_method_spider_arg(pipe.open_spider)+ if hasattr(pipe, "close_spider"):+ self.methods["close_spider"].appendleft(pipe.close_spider)+ self._check_mw_method_spider_arg(pipe.close_spider)if hasattr(pipe, "process_item"):self.methods["process_item"].append(pipe.process_item)+ self._check_mw_method_spider_arg(pipe.process_item)def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]:if spider:@@ -45,4 +55,36 @@ class ItemPipelineManager(MiddlewareManager):return deferred_from_coro(self.process_item_async(item))async def process_item_async(self, item: Any) -> Any:- return await self._process_chain("process_item", item, self._spider)+ return await self._process_chain("process_item", item, add_spider=True)++ def _process_parallel(self, methodname: str) -> Deferred[list[None]]:+ methods = cast("Iterable[Callable[..., None]]", self.methods[methodname])++ def get_dfd(method: Callable[..., None]) -> Deferred[None]:+ if method in self._mw_methods_requiring_spider:+ return maybeDeferred_coro(method, self._spider)+ return maybeDeferred_coro(method)++ dfds = [get_dfd(m) for m in methods]+ d: Deferred[list[tuple[bool, None]]] = DeferredList(+ dfds, fireOnOneErrback=True, consumeErrors=True+ )+ d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])++ def eb(failure: Failure) -> Failure:+ return failure.value.subFailure++ d2.addErrback(eb)+ return d2++ def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:+ if spider:+ self._warn_spider_arg("open_spider")+ self._set_compat_spider(spider)+ return self._process_parallel("open_spider")++ def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:+ if spider:+ self._warn_spider_arg("close_spider")+ self._set_compat_spider(spider)+ return self._process_parallel("close_spider")diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex a50150c59..fb65ff06a 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -165,12 +165,27 @@ class MediaPipeline(ABC):pipe._finish_init(crawler)return pipe- def open_spider(self, spider: Spider) -> None:- self.spiderinfo = self.SpiderInfo(spider)+ def open_spider(self, spider: Spider | None = None) -> None:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to MediaPipeline.open_spider()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ assert self.crawler.spider+ self.spiderinfo = self.SpiderInfo(self.crawler.spider)def process_item(- self, item: Any, spider: Spider+ self, item: Any, spider: Spider | None = None) -> Deferred[list[FileInfoOrError]]:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to MediaPipeline.process_item()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )info = self.spiderinforequests = arg_to_iter(self.get_media_requests(item, info))dlist = [self._process_request(r, info, item) for r in requests]diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pyindex 196b84d0d..a047809c3 100644--- a/scrapy/spidermiddlewares/base.py+++ b/scrapy/spidermiddlewares/base.py@@ -1,8 +1,10 @@from __future__ import annotations+import warningsfrom typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider+from scrapy.exceptions import ScrapyDeprecationWarningif TYPE_CHECKING:from collections.abc import AsyncIterator, Iterable@@ -53,15 +55,32 @@ class BaseSpiderMiddleware:yield odef process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider+ self, response: Response, result: Iterable[Any], spider: Spider | None = None) -> Iterable[Any]:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )for o in result:if (o := self._get_processed(o, response)) is not None:yield oasync def process_spider_output_async(- self, response: Response, result: AsyncIterator[Any], spider: Spider+ self,+ response: Response,+ result: AsyncIterator[Any],+ spider: Spider | None = None,) -> AsyncIterator[Any]:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )async for o in result:if (o := self._get_processed(o, response)) is not None:yield odiff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex fecdb0142..d23bca50c 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -7,8 +7,10 @@ See documentation in docs/topics/spider-middleware.rstfrom __future__ import annotationsimport logging+import warningsfrom typing import TYPE_CHECKING, Any+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.spidermiddlewares.base import BaseSpiderMiddlewareif TYPE_CHECKING:@@ -53,16 +55,33 @@ class DepthMiddleware(BaseSpiderMiddleware):return odef process_spider_output(- self, response: Response, result: Iterable[Any], spider: Spider+ self, response: Response, result: Iterable[Any], spider: Spider | None = None) -> Iterable[Any]:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to DepthMiddleware.process_spider_output()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )self._init_depth(response)- yield from super().process_spider_output(response, result, spider)+ yield from super().process_spider_output(response, result)async def process_spider_output_async(- self, response: Response, result: AsyncIterator[Any], spider: Spider+ self,+ response: Response,+ result: AsyncIterator[Any],+ spider: Spider | None = None,) -> AsyncIterator[Any]:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to DepthMiddleware.process_spider_output_async()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )self._init_depth(response)- async for o in super().process_spider_output_async(response, result, spider):+ async for o in super().process_spider_output_async(response, result):yield odef _init_depth(self, response: Response) -> None:diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 42619ec7f..b4c9c10bf 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rstfrom __future__ import annotationsimport logging+import warningsfrom typing import TYPE_CHECKING, Any-from scrapy.exceptions import IgnoreRequest+from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarningif TYPE_CHECKING:from collections.abc import Iterable@@ -35,9 +36,7 @@ class HttpError(IgnoreRequest):class HttpErrorMiddleware:- @classmethod- def from_crawler(cls, crawler: Crawler) -> Self:- return cls(crawler.settings)+ crawler: Crawlerdef __init__(self, settings: BaseSettings):self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL")@@ -45,7 +44,22 @@ class HttpErrorMiddleware:"HTTPERROR_ALLOWED_CODES")- def process_spider_input(self, response: Response, spider: Spider) -> None:+ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ o = cls(crawler.settings)+ o.crawler = crawler+ return o++ def process_spider_input(+ self, response: Response, spider: Spider | None = None+ ) -> None:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to HttpErrorMiddleware.process_spider_input()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )if 200 <= response.status < 300: # common casereturnmeta = response.meta@@ -57,25 +71,34 @@ class HttpErrorMiddleware:returnelse:allowed_statuses = getattr(- spider, "handle_httpstatus_list", self.handle_httpstatus_list+ self.crawler.spider,+ "handle_httpstatus_list",+ self.handle_httpstatus_list,)if response.status in allowed_statuses:returnraise HttpError(response, "Ignoring non-200 response")def process_spider_exception(- self, response: Response, exception: Exception, spider: Spider+ self, response: Response, exception: Exception, spider: Spider | None = None) -> Iterable[Any] | None:+ if spider is not None: # pragma: no cover+ warnings.warn(+ "Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"+ " is deprecated and the passed value is ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )if isinstance(exception, HttpError):- assert spider.crawler.stats- spider.crawler.stats.inc_value("httperror/response_ignored_count")- spider.crawler.stats.inc_value(+ assert self.crawler.stats+ self.crawler.stats.inc_value("httperror/response_ignored_count")+ self.crawler.stats.inc_value(f"httperror/response_ignored_status_count/{response.status}")logger.info("Ignoring response %(response)r: HTTP status code is not handled or not allowed",{"response": response},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)return []return Nonediff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex cbccbd244..7481e26ed 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -312,19 +312,6 @@ def process_chain(return d-async def _process_chain(- callables: Iterable[Callable[Concatenate[_T, _P], _T | Awaitable[_T]]],- input_: _T,- *a: _P.args,- **kw: _P.kwargs,-) -> _T:- """Chain the given (potentialy asynchronous) callables."""- result = input_- for callable_ in callables:- result = await ensure_awaitable(callable_(result, *a, **kw))- return result--def process_chain_both(callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],@@ -355,10 +342,15 @@ def process_parallel(input: _T, # noqa: A002*a: _P.args,**kw: _P.kwargs,-) -> Deferred[list[_T2]]:+) -> Deferred[list[_T2]]: # pragma: no cover"""Return a Deferred with the output of all successful calls to the givencallbacks"""+ warnings.warn(+ "process_parallel() is deprecated.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks]d: Deferred[list[tuple[bool, _T2]]] = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=Truediff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.pyindex c32aaf37d..e000a23db 100644--- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py+++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py@@ -16,7 +16,7 @@ class UppercasePipeline:def open_spider(self, spider):return deferred_from_coro(self._open_spider(spider))- def process_item(self, item, spider):+ def process_item(self, item):return {"url": item["url"].upper()}diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.pyindex 028e3a08a..eb2e1105c 100644--- a/tests/CrawlerProcess/asyncio_deferred_signal.py+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py@@ -16,7 +16,7 @@ class UppercasePipeline:def open_spider(self, spider):return deferred_from_coro(self._open_spider(spider))- def process_item(self, item, spider):+ def process_item(self, item):return {"url": item["url"].upper()}diff --git a/tests/pipelines.py b/tests/pipelines.pyindex d9fc12676..4f00d6919 100644--- a/tests/pipelines.py+++ b/tests/pipelines.py@@ -4,13 +4,13 @@ Some pipelines used for testingclass ZeroDivisionErrorPipeline:- def open_spider(self, spider):+ def open_spider(self):1 / 0- def process_item(self, item, spider):+ def process_item(self, item):return itemclass ProcessWithZeroDivisionErrorPipeline:- def process_item(self, item, spider):+ def process_item(self, item):1 / 0diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.pyindex 3e29c70ed..577ecd686 100644--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py@@ -2,7 +2,7 @@ class TestSpiderPipeline:def open_spider(self, spider):pass- def process_item(self, item, spider):+ def process_item(self, item):return item@@ -10,5 +10,5 @@ class TestSpiderExceptionPipeline:def open_spider(self, spider):raise RuntimeError("exception")- def process_item(self, item, spider):+ def process_item(self, item):return itemdiff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex fe7a807d4..7b119d7cf 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -153,7 +153,7 @@ import loggingclass MyPipeline:component_name = 'my_pipeline'- def process_item(self, item, spider):+ def process_item(self, item):logging.info('It Works!')return item""",diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex f21e52e52..426be13a7 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -246,7 +246,7 @@ class SkipMessagesLogFormatter(LogFormatter):class DropSomeItemsPipeline:drop = True- def process_item(self, item, spider):+ def process_item(self, item):if self.drop:self.drop = Falseraise DropItem("Ignoring item")diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex b6a07f872..6c3008b25 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -7,7 +7,6 @@ import pytestfrom scrapy import Spiderfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.middleware import MiddlewareManager-from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:@@ -57,7 +56,10 @@ class MyMiddlewareManager(MiddlewareManager):return [M1, MOff, M3]def _add_middleware(self, mw):- super()._add_middleware(mw)+ if hasattr(mw, "open_spider"):+ self.methods["open_spider"].append(mw.open_spider)+ if hasattr(mw, "close_spider"):+ self.methods["close_spider"].appendleft(mw.close_spider)if hasattr(mw, "process"):self.methods["process"].append(mw.process)@@ -105,73 +107,3 @@ def test_no_crawler() -> None:mwman = MyMiddlewareManager(m1, m2, m3)assert mwman.middlewares == (m1, m2, m3)assert mwman.crawler is None---def test_deprecated_spider_arg_no_crawler_spider(crawler: Crawler) -> None:- """Crawler is provided, but doesn't have a spider. The instance passed to the method is- ignored and raises a warning."""- mwman = MyMiddlewareManager(crawler=crawler)- with (- pytest.warns(- ScrapyDeprecationWarning,- match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",- ),- pytest.raises(- ValueError,- match="MyMiddlewareManager needs to access self.crawler.spider but it is None",- ),- ):- mwman.open_spider(DefaultSpider())---def test_deprecated_spider_arg_with_crawler(crawler: Crawler) -> None:- """Crawler is provided and has a spider, works. The instance passed to the method is ignored,- even if mismatched, but raises a warning."""- mwman = MyMiddlewareManager(crawler=crawler)- crawler.spider = crawler._create_spider("foo")- with pytest.warns(- ScrapyDeprecationWarning,- match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",- ):- mwman.open_spider(DefaultSpider())- with pytest.warns(- ScrapyDeprecationWarning,- match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",- ):- mwman.close_spider(DefaultSpider())---def test_deprecated_spider_arg_without_crawler() -> None:- """The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""- with pytest.warns(- ScrapyDeprecationWarning,- match="was called without the crawler argument",- ):- mwman = MyMiddlewareManager()- with pytest.warns(- ScrapyDeprecationWarning,- match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",- ):- mwman.open_spider(DefaultSpider())- with (- pytest.warns(- ScrapyDeprecationWarning,- match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",- ),- pytest.raises(RuntimeError, match="Different instances of Spider were passed"),- ):- mwman.close_spider(DefaultSpider())---def test_no_spider_arg_without_crawler() -> None:- """If no crawler and no spider arg, raise an error."""- with pytest.warns(- ScrapyDeprecationWarning,- match="was called without the crawler argument",- ):- mwman = MyMiddlewareManager()- with pytest.raises(- ValueError,- match="has no known Spider instance",- ):- mwman.open_spider()diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex a4cdaa5f8..db64589af 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -29,6 +29,7 @@ from scrapy.pipelines.files import (GCSFilesStore,S3FilesStore,)+from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.ftp import MockFTPServer@@ -78,10 +79,11 @@ class TestFilesPipeline:def setup_method(self):self.tempdir = mkdtemp()settings_dict = {"FILES_STORE": self.tempdir}- crawler = get_crawler(spidercls=None, settings_dict=settings_dict)+ crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)+ crawler.spider = crawler._create_spider()self.pipeline = FilesPipeline.from_crawler(crawler)self.pipeline.download_func = _mocked_download_func- self.pipeline.open_spider(None)+ self.pipeline.open_spider()def teardown_method(self):rmtree(self.tempdir)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 42ae897e8..cc1d73349 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -13,10 +13,10 @@ from scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileExceptionfrom scrapy.pipelines.media import MediaPipeline-from scrapy.spiders import Spiderfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_all+from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -48,12 +48,11 @@ class TestBaseMediaPipeline:settings = Nonedef setup_method(self):- spider_cls = Spider- self.spider = spider_cls("media.com")- crawler = get_crawler(spider_cls, self.settings)+ crawler = get_crawler(DefaultSpider, self.settings)+ crawler.spider = crawler._create_spider()self.pipe = self.pipeline_class.from_crawler(crawler)self.pipe.download_func = _mocked_download_func- self.pipe.open_spider(self.spider)+ self.pipe.open_spider()self.info = self.pipe.spiderinfoself.fingerprint = crawler.request_fingerprinter.fingerprint@@ -162,7 +161,7 @@ class TestBaseMediaPipeline:@inlineCallbacksdef test_default_process_item(self):item = {"name": "name"}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item is item@@ -216,7 +215,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):errback=self._errback,)item = {"requests": req}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item["results"] == [(True, {})]assert self.pipe._mockcalled == ["get_media_requests",@@ -236,7 +235,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):errback=self._errback,)item = {"requests": req}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert len(new_item["results"]) == 1assert new_item["results"][0][0] is Falseassert isinstance(new_item["results"][0][1], Failure)@@ -258,7 +257,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):fail = Failure(exc)req2 = Request("http://url2", meta={"response": fail})item = {"requests": [req1, req2]}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert len(new_item["results"]) == 2assert new_item["results"][0] == (True, {})assert new_item["results"][1][0] is False@@ -281,7 +280,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):# returns single Request (without callback)req = Request("http://url")item = {"requests": req} # pass a single item- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req) in self.info.downloaded@@ -289,7 +288,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):req1 = Request("http://url1")req2 = Request("http://url2")item = {"requests": iter([req1, req2])}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req1) in self.info.downloadedassert self.fingerprint(req2) in self.info.downloaded@@ -299,7 +298,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):rsp1 = Response("http://url1")req1 = Request("http://url1", meta={"response": rsp1})item = {"requests": req1}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item is itemassert new_item["results"] == [(True, {})]@@ -308,7 +307,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):req1.url, meta={"response": Response("http://donot.download.me")})item = {"requests": req2}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req1) == self.fingerprint(req2)assert new_item["results"] == [(True, {})]@@ -321,7 +320,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):req1.url, meta={"response": Response("http://donot.download.me")})item = {"requests": [req1, req2]}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item is itemassert new_item["results"] == [(True, {}), (True, {})]@@ -348,14 +347,14 @@ class TestMediaPipeline(TestBaseMediaPipeline):req1 = Request("http://url", meta={"response": rsp1_func})req2 = Request(req1.url, meta={"response": rsp2_func})item = {"requests": [req1, req2]}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item["results"] == [(True, {}), (True, {})]@inlineCallbacksdef test_use_media_to_download_result(self):req = Request("http://url", meta={"result": "ITSME"})item = {"requests": req}- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert new_item["results"] == [(True, "ITSME")]assert self.pipe._mockcalled == ["get_media_requests",@@ -555,12 +554,11 @@ class TestMediaFailedFailure:settings = Nonedef setup_method(self):- spider_cls = Spider- self.spider = spider_cls("media.com")- crawler = get_crawler(spider_cls, self.settings)+ crawler = get_crawler(DefaultSpider, self.settings)+ crawler.spider = crawler._create_spider()self.pipe = self.pipeline_class.from_crawler(crawler)self.pipe.download_func = _mocked_download_func- self.pipe.open_spider(self.spider)+ self.pipe.open_spider()self.info = self.pipe.spiderinfoself.fingerprint = crawler.request_fingerprinter.fingerprint@@ -587,7 +585,7 @@ class TestMediaFailedFailure:with pytest.warns(ScrapyDeprecationWarning, match="media_failed returned a Failure instance"):- new_item = yield self.pipe.process_item(item, self.spider)+ new_item = yield self.pipe.process_item(item)assert len(new_item["results"]) == 1assert new_item["results"][0][0] is Falseassert isinstance(new_item["results"][0][1], Failure)diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex 725ceccee..4b8007ead 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -4,6 +4,7 @@ import pytestfrom twisted.internet.defer import Deferred, inlineCallbacks, succeedfrom scrapy import Request, Spider, signals+from scrapy.crawler import Crawlerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncio import call_later@@ -19,17 +20,25 @@ from tests.mockserver.http import MockServerclass SimplePipeline:- def process_item(self, item, spider):+ def process_item(self, item):item["pipeline_passed"] = Truereturn item+class DeprecatedSpiderArgPipeline:+ def open_spider(self, spider):+ pass++ def close_spider(self, spider):+ pass++class DeferredPipeline:def cb(self, item):item["pipeline_passed"] = Truereturn item- def process_item(self, item, spider):+ def process_item(self, item):d = Deferred()d.addCallback(self.cb)d.callback(item)@@ -37,7 +46,7 @@ class DeferredPipeline:class AsyncDefPipeline:- async def process_item(self, item, spider):+ async def process_item(self, item):d = Deferred()call_later(0, d.callback, None)await maybe_deferred_to_future(d)@@ -46,7 +55,7 @@ class AsyncDefPipeline:class AsyncDefAsyncioPipeline:- async def process_item(self, item, spider):+ async def process_item(self, item):d = Deferred()loop = asyncio.get_event_loop()loop.call_later(0, d.callback, None)@@ -57,7 +66,7 @@ class AsyncDefAsyncioPipeline:class AsyncDefNotAsyncioPipeline:- async def process_item(self, item, spider):+ async def process_item(self, item):d1 = Deferred()from twisted.internet import reactor@@ -184,6 +193,9 @@ class TestCustomPipelineManager:def open_spider(self, spider): # pylint: disable=signature-differsreturn super().open_spider(spider)+ def close_spider(self, spider): # pylint: disable=signature-differs+ return super().close_spider(spider)+def process_item(self, item, spider): # pylint: disable=signature-differswith pytest.warns(ScrapyDeprecationWarning,@@ -212,6 +224,10 @@ class TestCustomPipelineManager:ScrapyDeprecationWarning,match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",+ ),pytest.warns(ScrapyDeprecationWarning,match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument",@@ -220,6 +236,10 @@ class TestCustomPipelineManager:ScrapyDeprecationWarning,match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated",),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to CustomPipelineManager.close_spider\(\) is deprecated",+ ),pytest.warns(ScrapyDeprecationWarning,match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",@@ -254,7 +274,7 @@ class TestCustomPipelineManager:def process_item(self, item, spider):for pipeline in self.pipelines:- item = pipeline.process_item(item, spider)+ item = pipeline.process_item(item)return succeed(item)items = []@@ -294,3 +314,128 @@ class TestCustomPipelineManager:await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))assert len(items) == 1+++class TestMiddlewareManagerSpider:+ """Tests for the deprecated spider arg handling in MiddlewareManager.++ Here because MiddlewareManager doesn't have methods that could take a spider arg."""++ @pytest.fixture+ def crawler(self) -> Crawler:+ return get_crawler(Spider)++ def test_deprecated_spider_arg_no_crawler_spider(self, crawler: Crawler) -> None:+ """Crawler is provided, but doesn't have a spider. The instance passed to the method is+ ignored and raises a warning."""+ mwman = ItemPipelineManager(crawler=crawler)+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",+ ),+ ):+ mwman._add_middleware(DeprecatedSpiderArgPipeline())+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",+ ),+ pytest.raises(+ ValueError,+ match="ItemPipelineManager needs to access self.crawler.spider but it is None",+ ),+ ):+ mwman.open_spider(DefaultSpider())+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",+ ),+ pytest.raises(+ ValueError,+ match="ItemPipelineManager needs to access self.crawler.spider but it is None",+ ),+ ):+ mwman.close_spider(DefaultSpider())++ def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:+ """Crawler is provided and has a spider, works. The instance passed to the method is ignored,+ even if mismatched, but raises a warning."""+ mwman = ItemPipelineManager(crawler=crawler)+ crawler.spider = crawler._create_spider("foo")+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",+ ):+ mwman.open_spider(DefaultSpider())+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",+ ):+ mwman.close_spider(DefaultSpider())++ def test_deprecated_spider_arg_without_crawler(self) -> None:+ """The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="was called without the crawler argument",+ ):+ mwman = ItemPipelineManager()+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",+ ),+ ):+ mwman._add_middleware(DeprecatedSpiderArgPipeline())+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",+ ):+ mwman.open_spider(DefaultSpider())+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",+ ),+ pytest.raises(+ RuntimeError, match="Different instances of Spider were passed"+ ),+ ):+ mwman.close_spider(DefaultSpider())+ mwman.close_spider()++ def test_no_spider_arg_without_crawler(self) -> None:+ """If no crawler and no spider arg, raise an error."""+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="was called without the crawler argument",+ ):+ mwman = ItemPipelineManager()+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",+ ),+ ):+ mwman._add_middleware(DeprecatedSpiderArgPipeline())+ with (+ pytest.raises(+ ValueError,+ match="has no known Spider instance",+ ),+ ):+ mwman.open_spider()diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 041fc9588..0f31af8ad 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -33,12 +33,12 @@ class InjectArgumentsSpiderMiddleware:request.cb_kwargs["from_process_start"] = Trueyield request- def process_spider_input(self, response, spider):+ def process_spider_input(self, response):request = response.requestif request.callback.__name__ == "parse_spider_mw":request.cb_kwargs["from_process_spider_input"] = True- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):for element in result:if (isinstance(element, Request)diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 76609ac52..c1f358bd2 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -451,7 +451,7 @@ class TestSettings:def test_passing_objects_as_values(self):class TestPipeline:- def process_item(self, i, s):+ def process_item(self, i):return isettings = Settings(@@ -471,7 +471,7 @@ class TestSettings:assert priority == 800assert mypipeline == TestPipelineassert isinstance(mypipeline(), TestPipeline)- assert mypipeline().process_item("item", None) == "item"+ assert mypipeline().process_item("item") == "item"myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")assert myhandler == FileDownloadHandlerdiff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 2ef810852..8fd33eb13 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -10,7 +10,7 @@ from testfixtures import LogCapturefrom twisted.internet import deferfrom scrapy.core.spidermw import SpiderMiddlewareManager-from scrapy.exceptions import _InvalidOutput+from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.asyncgen import collect_asyncgen@@ -54,7 +54,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):@deferred_f_from_coro_fasync def test_invalid_process_spider_input(self):class InvalidProcessSpiderInputMiddleware:- def process_spider_input(self, response, spider):+ def process_spider_input(self, response):return 1self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())@@ -68,7 +68,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):@deferred_f_from_coro_fasync def test_invalid_process_spider_output(self):class InvalidProcessSpiderOutputMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):return 1self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())@@ -82,11 +82,11 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):@deferred_f_from_coro_fasync def test_invalid_process_spider_exception(self):class InvalidProcessSpiderOutputExceptionMiddleware:- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):return 1class RaiseExceptionProcessSpiderOutputMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):raise RuntimeErrorself.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())@@ -101,11 +101,11 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):@deferred_f_from_coro_fasync def test_process_spider_exception_return_none(self):class ProcessSpiderExceptionReturnNoneMiddleware:- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):return Noneclass RaiseExceptionProcessSpiderOutputMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):1 / 0self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())@@ -191,34 +191,34 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):class ProcessSpiderOutputSimpleMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):yield from resultclass ProcessSpiderOutputAsyncGenMiddleware:- async def process_spider_output(self, response, result, spider):+ async def process_spider_output(self, response, result):async for r in result:yield rclass ProcessSpiderOutputUniversalMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):yield from result- async def process_spider_output_async(self, response, result, spider):+ async def process_spider_output_async(self, response, result):async for r in result:yield rclass ProcessSpiderExceptionSimpleIterableMiddleware:- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):yield {"foo": 1}yield {"foo": 2}yield {"foo": 3}class ProcessSpiderExceptionAsyncIteratorMiddleware:- async def process_spider_exception(self, response, exception, spider):+ async def process_spider_exception(self, response, exception):yield {"foo": 1}d = defer.Deferred()call_later(0, d.callback, None)@@ -315,12 +315,12 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):class ProcessSpiderOutputNonIterableMiddleware:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):returnclass ProcessSpiderOutputCoroutineMiddleware:- async def process_spider_output(self, response, result, spider):+ async def process_spider_output(self, response, result):return result@@ -384,23 +384,23 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):class UniversalMiddlewareNoSync:- async def process_spider_output_async(self, response, result, spider):+ async def process_spider_output_async(self, response, result):yieldclass UniversalMiddlewareBothSync:- def process_spider_output(self, response, result, spider):+ def process_spider_output(self, response, result):yield- def process_spider_output_async(self, response, result, spider):+ def process_spider_output_async(self, response, result):yieldclass UniversalMiddlewareBothAsync:- async def process_spider_output(self, response, result, spider):+ async def process_spider_output(self, response, result):yield- async def process_spider_output_async(self, response, result, spider):+ async def process_spider_output_async(self, response, result):yield@@ -600,3 +600,34 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):async def test_exc_async_simple(self):"""Async exc mw -> simple output mw; cannot work as downgrading is not supported"""await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)+++class TestDeprecatedSpiderArg(TestSpiderMiddleware):+ @deferred_f_from_coro_f+ async def test_deprecated_spider_arg(self):+ class DeprecatedSpiderArgMiddleware:+ def process_spider_input(self, response, spider):+ return None++ def process_spider_output(self, response, result, spider):+ 1 / 0++ def process_spider_exception(self, response, exception, spider):+ return []++ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"process_spider_input\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"process_spider_output\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"process_spider_exception\(\) requires a spider argument",+ ),+ ):+ self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())+ await self._scrape_response()diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex 03446db7e..00e547305 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -21,12 +21,6 @@ def crawler() -> Crawler:return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})-def spider(crawler: Crawler) -> Spider:- crawler.spider = crawler._create_spider("scrapytest.org")- return crawler.spider--@pytest.fixturedef stats(crawler: Crawler) -> Generator[StatsCollector]:assert crawler.stats is not None@@ -42,15 +36,13 @@ def mw(crawler: Crawler) -> DepthMiddleware:return DepthMiddleware.from_crawler(crawler)-def test_process_spider_output(- mw: DepthMiddleware, stats: StatsCollector, spider: Spider-) -> None:+def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:req = Request("http://scrapytest.org")resp = Response("http://scrapytest.org")resp.request = reqresult = [Request("http://scrapytest.org")]- out = list(mw.process_spider_output(resp, result, spider))+ out = list(mw.process_spider_output(resp, result))assert out == resultrdc = stats.get_value("request_depth_count/1")@@ -58,7 +50,7 @@ def test_process_spider_output(req.meta["depth"] = 1- out2 = list(mw.process_spider_output(resp, result, spider))+ out2 = list(mw.process_spider_output(resp, result))assert not out2rdm = stats.get_value("request_depth_max")diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex b41ae986e..9899a0cd7 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -7,9 +7,8 @@ from testfixtures import LogCapturefrom twisted.internet.defer import inlineCallbacksfrom scrapy.http import Request, Response-from scrapy.settings import Settingsfrom scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware-from scrapy.spiders import Spider+from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import MockServerSpider@@ -57,12 +56,6 @@ def _response(request: Request, status_code: int) -> Response:return Response(request.url, status=status_code, request=request)-def spider() -> Spider:- crawler = get_crawler(Spider)- return Spider.from_crawler(crawler, name="foo")--@pytest.fixturedef res200() -> Response:return _response(req, 200)@@ -81,36 +74,35 @@ def res404() -> Response:class TestHttpErrorMiddleware:@pytest.fixturedef mw(self) -> HttpErrorMiddleware:- return HttpErrorMiddleware(Settings({}))+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ return HttpErrorMiddleware.from_crawler(crawler)def test_process_spider_input(- self,- mw: HttpErrorMiddleware,- spider: Spider,- res200: Response,- res404: Response,+ self, mw: HttpErrorMiddleware, res200: Response, res404: Response) -> None:- mw.process_spider_input(res200, spider)+ mw.process_spider_input(res200)with pytest.raises(HttpError):- mw.process_spider_input(res404, spider)+ mw.process_spider_input(res404)def test_process_spider_exception(- self, mw: HttpErrorMiddleware, spider: Spider, res404: Response+ self, mw: HttpErrorMiddleware, res404: Response) -> None:- assert mw.process_spider_exception(res404, HttpError(res404), spider) == []- assert mw.process_spider_exception(res404, Exception(), spider) is None+ assert mw.process_spider_exception(res404, HttpError(res404)) == []+ assert mw.process_spider_exception(res404, Exception()) is Nonedef test_handle_httpstatus_list(- self, mw: HttpErrorMiddleware, spider: Spider, res404: Response+ self, mw: HttpErrorMiddleware, res404: Response) -> None:request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})res = _response(request, 404)- mw.process_spider_input(res, spider)+ mw.process_spider_input(res)- spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]- mw.process_spider_input(res404, spider)+ assert mw.crawler.spider+ mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]+ mw.process_spider_input(res404)class TestHttpErrorMiddlewareSettings:@@ -118,76 +110,71 @@ class TestHttpErrorMiddlewareSettings:@pytest.fixturedef mw(self) -> HttpErrorMiddleware:- return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})+ crawler.spider = crawler._create_spider()+ return HttpErrorMiddleware.from_crawler(crawler)def test_process_spider_input(self,mw: HttpErrorMiddleware,- spider: Spider,res200: Response,res402: Response,res404: Response,) -> None:- mw.process_spider_input(res200, spider)+ mw.process_spider_input(res200)with pytest.raises(HttpError):- mw.process_spider_input(res404, spider)- mw.process_spider_input(res402, spider)+ mw.process_spider_input(res404)+ mw.process_spider_input(res402)- def test_meta_overrides_settings(- self, mw: HttpErrorMiddleware, spider: Spider- ) -> None:+ def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})res404 = _response(request, 404)res402 = _response(request, 402)- mw.process_spider_input(res404, spider)+ mw.process_spider_input(res404)with pytest.raises(HttpError):- mw.process_spider_input(res402, spider)+ mw.process_spider_input(res402)def test_spider_override_settings(- self,- mw: HttpErrorMiddleware,- spider: Spider,- res402: Response,- res404: Response,+ self, mw: HttpErrorMiddleware, res402: Response, res404: Response) -> None:- spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]- mw.process_spider_input(res404, spider)+ assert mw.crawler.spider+ mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]+ mw.process_spider_input(res404)with pytest.raises(HttpError):- mw.process_spider_input(res402, spider)+ mw.process_spider_input(res402)class TestHttpErrorMiddlewareHandleAll:@pytest.fixturedef mw(self) -> HttpErrorMiddleware:- return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})+ crawler.spider = crawler._create_spider()+ return HttpErrorMiddleware.from_crawler(crawler)def test_process_spider_input(self,mw: HttpErrorMiddleware,- spider: Spider,res200: Response,res404: Response,) -> None:- mw.process_spider_input(res200, spider)- mw.process_spider_input(res404, spider)+ mw.process_spider_input(res200)+ mw.process_spider_input(res404)- def test_meta_overrides_settings(- self, mw: HttpErrorMiddleware, spider: Spider- ) -> None:+ def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})res404 = _response(request, 404)res402 = _response(request, 402)- mw.process_spider_input(res404, spider)+ mw.process_spider_input(res404)with pytest.raises(HttpError):- mw.process_spider_input(res402, spider)+ mw.process_spider_input(res402)- def test_httperror_allow_all_false(self, spider: Spider) -> None:+ def test_httperror_allow_all_false(self) -> None:crawler = get_crawler(_HttpErrorSpider)mw = HttpErrorMiddleware.from_crawler(crawler)request_httpstatus_false = Request(@@ -200,8 +187,8 @@ class TestHttpErrorMiddlewareHandleAll:res402 = _response(request_httpstatus_true, 402)with pytest.raises(HttpError):- mw.process_spider_input(res404, spider)- mw.process_spider_input(res402, spider)+ mw.process_spider_input(res404)+ mw.process_spider_input(res402)class TestHttpErrorMiddlewareIntegrational:@@ -218,7 +205,7 @@ class TestHttpErrorMiddlewareIntegrational:def test_middleware_works(self):crawler = get_crawler(_HttpErrorSpider)yield crawler.crawl(mockserver=self.mockserver)- assert not crawler.spider.skipped, crawler.spider.skipped+ assert not crawler.spider.skippedassert crawler.spider.parsed == {"200"}assert crawler.spider.failed == {"404", "402", "500"}diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 3f2b2057b..1808e087c 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -6,18 +6,27 @@ from scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer-class LogExceptionMiddleware:- def process_spider_exception(self, response, exception, spider):- spider.logger.info(+class _BaseSpiderMiddleware:+ def __init__(self, crawler):+ self.crawler = crawler++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)+++class LogExceptionMiddleware(_BaseSpiderMiddleware):+ def process_spider_exception(self, response, exception):+ self.crawler.spider.logger.info("Middleware: %s exception caught", exception.__class__.__name__)# ================================================================================# (0) recover from an exception on a spider callback-class RecoveryMiddleware:- def process_spider_exception(self, response, exception, spider):- spider.logger.info(+class RecoveryMiddleware(_BaseSpiderMiddleware):+ def process_spider_exception(self, response, exception):+ self.crawler.spider.logger.info("Middleware: %s exception caught", exception.__class__.__name__)return [@@ -55,9 +64,9 @@ class RecoveryAsyncGenSpider(RecoverySpider):# ================================================================================# (1) exceptions from a spider middleware's process_spider_input method-class FailProcessSpiderInputMiddleware:- def process_spider_input(self, response, spider):- spider.logger.info("Middleware: will raise IndexError")+class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):+ def process_spider_input(self, response):+ self.crawler.spider.logger.info("Middleware: will raise IndexError")raise IndexError@@ -159,27 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS# ================================================================================# (4) exceptions from a middleware process_spider_output method (generator)-class _GeneratorDoNothingMiddleware:- def process_spider_output(self, response, result, spider):+class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):+ def process_spider_output(self, response, result):for r in result:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")yield r- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):method = f"{self.__class__.__name__}.process_spider_exception"- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)+ self.crawler.spider.logger.info(+ "%s: %s caught", method, exception.__class__.__name__+ )-class GeneratorFailMiddleware:- def process_spider_output(self, response, result, spider):+class GeneratorFailMiddleware(_BaseSpiderMiddleware):+ def process_spider_output(self, response, result):for r in result:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")yield rraise LookupError- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):method = f"{self.__class__.__name__}.process_spider_exception"- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)+ self.crawler.spider.logger.info(+ "%s: %s caught", method, exception.__class__.__name__+ )yield {"processed": [method]}@@ -187,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):pass-class GeneratorRecoverMiddleware:- def process_spider_output(self, response, result, spider):+class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):+ def process_spider_output(self, response, result):for r in result:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")yield r- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):method = f"{self.__class__.__name__}.process_spider_exception"- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)+ self.crawler.spider.logger.info(+ "%s: %s caught", method, exception.__class__.__name__+ )yield {"processed": [method]}@@ -226,30 +241,34 @@ class GeneratorOutputChainSpider(Spider):# (5) exceptions from a middleware process_spider_output method (not generator)-class _NotGeneratorDoNothingMiddleware:- def process_spider_output(self, response, result, spider):+class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware):+ def process_spider_output(self, response, result):out = []for r in result:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")out.append(r)return out- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):method = f"{self.__class__.__name__}.process_spider_exception"- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)+ self.crawler.spider.logger.info(+ "%s: %s caught", method, exception.__class__.__name__+ )-class NotGeneratorFailMiddleware:- def process_spider_output(self, response, result, spider):+class NotGeneratorFailMiddleware(_BaseSpiderMiddleware):+ def process_spider_output(self, response, result):out = []for r in result:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")out.append(r)raise ReferenceError- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):method = f"{self.__class__.__name__}.process_spider_exception"- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)+ self.crawler.spider.logger.info(+ "%s: %s caught", method, exception.__class__.__name__+ )return [{"processed": [method]}]@@ -257,17 +276,19 @@ class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddlewapass-class NotGeneratorRecoverMiddleware:- def process_spider_output(self, response, result, spider):+class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware):+ def process_spider_output(self, response, result):out = []for r in result:r["processed"].append(f"{self.__class__.__name__}.process_spider_output")out.append(r)return out- def process_spider_exception(self, response, exception, spider):+ def process_spider_exception(self, response, exception):method = f"{self.__class__.__name__}.process_spider_exception"- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)+ self.crawler.spider.logger.info(+ "%s: %s caught", method, exception.__class__.__name__+ )return [{"processed": [method]}]diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 895df1804..658351999 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -61,11 +61,11 @@ class TestRefererMiddleware:def get_response(self, origin: str) -> Response:return Response(origin, headers=self.resp_headers)- def test(self, mw: RefererMiddleware, spider: Spider) -> None:+ def test(self, mw: RefererMiddleware) -> None:for origin, target, referrer in self.scenarii:response = self.get_response(origin)request = self.get_request(target)- out = list(mw.process_spider_output(response, [request], spider))+ out = list(mw.process_spider_output(response, [request]))assert out[0].headers.get("Referer") == referrer@@ -1036,7 +1036,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):response = self.get_response(parent)request = self.get_request(target)- out = list(referrermw.process_spider_output(response, [request], spider))+ out = list(referrermw.process_spider_output(response, [request]))assert out[0].headers.get("Referer") == init_referrerfor status, url in redirections:diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex ca65529a3..750ae3b07 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -27,11 +27,6 @@ def crawler() -> Crawler:return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength})-def spider(crawler: Crawler) -> Spider:- return crawler._create_spider("foo")--@pytest.fixturedef stats(crawler: Crawler) -> StatsCollector:assert crawler.stats is not None@@ -43,22 +38,19 @@ def mw(crawler: Crawler) -> UrlLengthMiddleware:return UrlLengthMiddleware.from_crawler(crawler)-def process_spider_output(mw: UrlLengthMiddleware, spider: Spider) -> list[Request]:- return list(mw.process_spider_output(response, reqs, spider))+def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:+ return list(mw.process_spider_output(response, reqs))-def test_middleware_works(mw: UrlLengthMiddleware, spider: Spider) -> None:- assert process_spider_output(mw, spider) == [short_url_req]+def test_middleware_works(mw: UrlLengthMiddleware) -> None:+ assert process_spider_output(mw) == [short_url_req]def test_logging(- stats: StatsCollector,- mw: UrlLengthMiddleware,- spider: Spider,- caplog: pytest.LogCaptureFixture,+ stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture) -> None:with caplog.at_level(INFO):- process_spider_output(mw, spider)+ process_spider_output(mw)ric = stats.get_value("urllength/request_ignored_count")assert ric == 1assert f"Ignoring link (url length > {maxlength})" in caplog.textdiff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 7570fb96c..df7f1a20e 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, inlineCallbacks, succeedfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.defer import (- _process_chain,aiter_errback,deferred_f_from_coro_f,deferred_from_coro,@@ -19,7 +18,6 @@ from scrapy.utils.defer import (maybe_deferred_to_future,mustbe_deferred,parallel_async,- process_parallel,)if TYPE_CHECKING:@@ -85,26 +83,6 @@ def eb1(failure, arg1, arg2):return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"-class TestDeferUtils:- @deferred_f_from_coro_f- async def test_process_chain(self):- x = await _process_chain([cb1, cb2, cb3], "res", "v1", "v2")- assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)"-- with pytest.raises(TypeError):- await _process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")-- @inlineCallbacks- def test_process_parallel(self):- x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")- assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]-- @inlineCallbacks- def test_process_parallel_failure(self):- with pytest.raises(TypeError):- yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")--class TestIterErrback:def test_iter_errback_good(self):def itergood() -> Generator[int, None, None]: