Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate spider args of middlewares and pipelines. (#7006)

Andrey Rakhmatullined63fa94d698
diff --git a/docs/faq.rst b/docs/faq.rst
index 1d09a0e63..fdf0582de 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -349,7 +349,7 @@ method for this purpose. For example:
class MultiplyItemsMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
for item_or_request in result:
if isinstance(item_or_request, Request):
continue
diff --git a/docs/news.rst b/docs/news.rst
index 92320ac6a..6a019e2fb 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -44,6 +44,20 @@ Backward-incompatible changes
- :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`
+- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for
+ handling ``open_spider()`` and ``close_spider()`` component methods. As
+ this code was only used for pipelines it was moved into
+ :class:`scrapy.pipelines.ItemPipelineManager`. This change should only
+ affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`.
+ The following code was moved:
+
+ - ``scrapy.middleware.MiddlewareManager.open_spider()``
+
+ - ``scrapy.middleware.MiddlewareManager.close_spider()``
+
+ - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that
+ processes ``open_spider()`` and ``close_spider()`` component methods.
+
- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request`
now returns a coroutine, previously it returned a
:class:`~twisted.internet.defer.Deferred` object or ``None``. The
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index b9d780528..fdd3dcd87 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -191,7 +191,7 @@ shorter and cleaner:
adapter["field"] = data
return item
- def process_item(self, item, spider):
+ def process_item(self, item):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item)
@@ -205,7 +205,7 @@ becomes:
class DbPipeline:
- async def process_item(self, item, spider):
+ async def process_item(self, item):
adapter = ItemAdapter(item)
adapter["field"] = await db.get_some_data(adapter["id"])
return item
@@ -421,12 +421,12 @@ For example:
.. code-block:: python
class UniversalSpiderMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
for r in result:
# ... do something with r
yield r
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
async for r in result:
# ... do something with r
yield r
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 2fbe3f75d..d3186ef5c 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -67,7 +67,7 @@ value of one of their fields:
self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0]
- def process_item(self, item, spider):
+ def process_item(self, item):
exporter = self._exporter_for_item(item)
exporter.export_item(item)
return item
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 13a5b88f7..e67cf06c8 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -26,7 +26,7 @@ Writing your own item pipeline
Each item pipeline is a :ref:`component <topics-components>` that must
implement the following method:
-.. method:: process_item(self, item, spider)
+.. method:: process_item(self, item)
This method is called for every item pipeline component.
@@ -42,25 +42,16 @@ implement the following method:
:param item: the scraped item
:type item: :ref:`item object <item-types>`
- :param spider: the spider which scraped the item
- :type spider: :class:`~scrapy.Spider` object
-
Additionally, they may also implement the following methods:
-.. method:: open_spider(self, spider)
+.. method:: open_spider(self)
This method is called when the spider is opened.
- :param spider: the spider which was opened
- :type spider: :class:`~scrapy.Spider` object
-
-.. method:: close_spider(self, spider)
+.. method:: close_spider(self)
This method is called when the spider is closed.
- :param spider: the spider which was closed
- :type spider: :class:`~scrapy.Spider` object
-
Item pipeline example
=====================
@@ -82,7 +73,7 @@ contain a price:
class PricePipeline:
vat_factor = 1.15
- def process_item(self, item, spider):
+ def process_item(self, item):
adapter = ItemAdapter(item)
if adapter.get("price"):
if adapter.get("price_excludes_vat"):
@@ -107,13 +98,13 @@ format:
class JsonWriterPipeline:
- def open_spider(self, spider):
+ def open_spider(self):
self.file = open("items.jsonl", "w")
- def close_spider(self, spider):
+ def close_spider(self):
self.file.close()
- def process_item(self, item, spider):
+ def process_item(self, item):
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line)
return item
@@ -153,14 +144,14 @@ The main point of this example is to show how to :ref:`get the crawler
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
)
- def open_spider(self, spider):
+ def open_spider(self):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
- def close_spider(self, spider):
+ def close_spider(self):
self.client.close()
- def process_item(self, item, spider):
+ def process_item(self, item):
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item
@@ -198,12 +189,19 @@ item.
SPLASH_URL = "http://localhost:8050/render.png?url={}"
- async def process_item(self, item, spider):
+ def __init__(crawler):
+ self.crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ async def process_item(self, item):
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
- response = await spider.crawler.engine.download_async(request)
+ response = await self.crawler.engine.download_async(request)
if response.status != 200:
# Error happened, return item.
@@ -238,7 +236,7 @@ returns multiples items with the same id:
def __init__(self):
self.ids_seen = set()
- def process_item(self, item, spider):
+ def process_item(self, item):
adapter = ItemAdapter(item)
if adapter["id"] in self.ids_seen:
raise DropItem(f"Item ID already seen: {adapter['id']}")
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e9519eef..9dc26d703 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -554,7 +554,7 @@ When writing an item pipeline, you can force a different log level by setting
class MyPipeline:
- def process_item(self, item, spider):
+ def process_item(self, item):
if not item.get("price"):
raise DropItem("Missing price data", log_level="INFO")
return item
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 2eb59fe44..0f81e409e 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -94,7 +94,7 @@ one or more of these methods:
def process_start_requests(self, start, spider):
yield from start
- .. method:: process_spider_input(response, spider)
+ .. method:: process_spider_input(response)
This method is called for each response that goes through the spider
middleware and into the spider, for processing.
@@ -116,11 +116,7 @@ one or more of these methods:
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object
- :param spider: the spider for which this response is intended
- :type spider: :class:`~scrapy.Spider` object
-
-
- .. method:: process_spider_output(response, result, spider)
+ .. method:: process_spider_output(response, result)
This method is called with the results returned from the Spider, after
it has processed the response.
@@ -149,10 +145,7 @@ one or more of these methods:
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item objects <topics-items>`
- :param spider: the spider whose result is being processed
- :type spider: :class:`~scrapy.Spider` object
-
- .. method:: process_spider_output_async(response, result, spider)
+ .. method:: process_spider_output_async(response, result)
:async:
.. versionadded:: 2.7
@@ -161,7 +154,7 @@ one or more of these methods:
which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`.
- .. method:: process_spider_exception(response, exception, spider)
+ .. method:: process_spider_exception(response, exception)
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
@@ -186,8 +179,6 @@ one or more of these methods:
:param exception: the exception raised
:type exception: :exc:`Exception` object
- :param spider: the spider which raised the exception
- :type spider: :class:`~scrapy.Spider` object
Base class for custom spider middlewares
----------------------------------------
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 9f92e906d..75246710d 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -117,9 +117,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
def _add_middleware(self, mw: Any) -> None:
- super()._add_middleware(mw)
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
+ self._check_mw_method_spider_arg(mw.process_spider_input)
+
if self._use_start_requests:
if hasattr(mw, "process_start_requests"):
self.methods["process_start_requests"].appendleft(
@@ -127,10 +128,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
elif hasattr(mw, "process_start"):
self.methods["process_start"].appendleft(mw.process_start)
+
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
self.methods["process_spider_output"].appendleft(process_spider_output)
+ if callable(process_spider_output):
+ self._check_mw_method_spider_arg(process_spider_output)
+ elif isinstance(process_spider_output, tuple):
+ for m in process_spider_output:
+ self._check_mw_method_spider_arg(m)
+
process_spider_exception = getattr(mw, "process_spider_exception", None)
self.methods["process_spider_exception"].appendleft(process_spider_exception)
+ if process_spider_exception is not None:
+ self._check_mw_method_spider_arg(process_spider_exception)
async def _process_spider_input(
self,
@@ -141,7 +151,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method in self.methods["process_spider_input"]:
method = cast("Callable", method)
try:
- result = method(response=response, spider=self._spider)
+ if method in self._mw_methods_requiring_spider:
+ result = method(response=response, spider=self._spider)
+ else:
+ result = method(response=response)
if result is not None:
msg = (
f"{global_object_name(method)} must return None "
@@ -212,7 +225,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
if method is None:
continue
method = cast("Callable", method)
- result = method(response=response, exception=exception, spider=self._spider)
+ if method in self._mw_methods_requiring_spider:
+ result = method(
+ response=response, exception=exception, spider=self._spider
+ )
+ else:
+ result = method(response=response, exception=exception)
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
@@ -298,7 +316,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
recovered = MutableChain(recovered_collected)
# might fail directly if the output value is not a generator
- result = method(response=response, result=result, spider=self._spider)
+ if method in self._mw_methods_requiring_spider:
+ result = method(
+ response=response, result=result, spider=self._spider
+ )
+ else:
+ result = method(response=response, result=result)
except Exception as ex:
exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (
self._process_spider_exception(response, ex, method_index + 1)
@@ -421,7 +444,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if self._use_start_requests:
sync_start = iter(self._spider.start_requests())
sync_start = await self._process_chain(
- "process_start_requests", sync_start, self._spider
+ "process_start_requests", sync_start, always_add_spider=True
)
start: AsyncIterator[Any] = as_async_generator(sync_start)
else:
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 8d2ade120..be41b52e4 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -8,7 +8,8 @@ from collections import defaultdict, deque
from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.utils.defer import _process_chain, process_parallel
+from scrapy.utils.defer import ensure_awaitable
+from scrapy.utils.deprecate import argument_is_required
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import global_object_name
@@ -31,7 +32,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
-_T2 = TypeVar("_T2")
class MiddlewareManager(ABC):
@@ -56,6 +56,7 @@ class MiddlewareManager(ABC):
self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = (
defaultdict(deque)
)
+ self._mw_methods_requiring_spider: set[Callable] = set()
for mw in middlewares:
self._add_middleware(mw)
@@ -158,34 +159,53 @@ class MiddlewareManager(ABC):
)
return cls(*middlewares, crawler=crawler)
- def _add_middleware(self, mw: Any) -> None:
- if hasattr(mw, "open_spider"):
- self.methods["open_spider"].append(mw.open_spider)
- if hasattr(mw, "close_spider"):
- self.methods["close_spider"].appendleft(mw.close_spider)
+ def _add_middleware(self, mw: Any) -> None: # noqa: B027
+ pass
- def _process_parallel(
- self, methodname: str, obj: _T, *args: Any
- ) -> Deferred[list[_T2]]:
+ def _check_mw_method_spider_arg(self, method: Callable) -> None:
+ if argument_is_required(method, "spider"):
+ warnings.warn(
+ f"{method.__qualname__}() requires a spider argument,"
+ f" this is deprecated and the argument will not be passed in future Scrapy versions."
+ f" If you need to access the spider instance you can save the crawler instance"
+ f" passed to from_crawler() and use its spider attribute.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ self._mw_methods_requiring_spider.add(method)
+
+ async def _process_chain(
+ self,
+ methodname: str,
+ obj: _T,
+ *args: Any,
+ add_spider: bool = False,
+ always_add_spider: bool = False,
+ ) -> _T:
methods = cast(
- "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname]
+ "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
+ )
+ for method in methods:
+ if always_add_spider or (
+ add_spider and method in self._mw_methods_requiring_spider
+ ):
+ obj = await ensure_awaitable(method(obj, *(*args, self._spider)))
+ else:
+ obj = await ensure_awaitable(method(obj, *args))
+ return obj
+
+ def open_spider(
+ self, spider: Spider | None = None
+ ) -> Deferred[list[None]]: # pragma: no cover
+ raise NotImplementedError(
+ "MiddlewareManager.open_spider() is no longer implemented"
+ " and will be removed in a future Scrapy version."
)
- return process_parallel(methods, obj, *args)
- async def _process_chain(self, methodname: str, obj: _T, *args: Any) -> _T:
- methods = cast(
- "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
+ def close_spider(
+ self, spider: Spider | None = None
+ ) -> Deferred[list[None]]: # pragma: no cover
+ raise NotImplementedError(
+ "MiddlewareManager.close_spider() is no longer implemented"
+ " and will be removed in a future Scrapy version."
)
- return await _process_chain(methods, obj, *args)
-
- def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
- if spider:
- self._warn_spider_arg("open_spider")
- self._set_compat_spider(spider)
- return self._process_parallel("open_spider", self._spider)
-
- def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
- if spider:
- self._warn_spider_arg("close_spider")
- self._set_compat_spider(spider)
- return self._process_parallel("close_spider", self._spider)
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 8fe40ab13..b7ec928e8 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -7,16 +7,20 @@ See documentation in docs/item-pipeline.rst
from __future__ import annotations
import warnings
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Callable, cast
+
+from twisted.internet.defer import Deferred, DeferredList
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, maybeDeferred_coro
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
- from twisted.internet.defer import Deferred
+ from collections.abc import Iterable
+
+ from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.settings import Settings
@@ -30,9 +34,15 @@ class ItemPipelineManager(MiddlewareManager):
return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
def _add_middleware(self, pipe: Any) -> None:
- super()._add_middleware(pipe)
+ if hasattr(pipe, "open_spider"):
+ self.methods["open_spider"].append(pipe.open_spider)
+ self._check_mw_method_spider_arg(pipe.open_spider)
+ if hasattr(pipe, "close_spider"):
+ self.methods["close_spider"].appendleft(pipe.close_spider)
+ self._check_mw_method_spider_arg(pipe.close_spider)
if hasattr(pipe, "process_item"):
self.methods["process_item"].append(pipe.process_item)
+ self._check_mw_method_spider_arg(pipe.process_item)
def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]:
if spider:
@@ -45,4 +55,36 @@ class ItemPipelineManager(MiddlewareManager):
return deferred_from_coro(self.process_item_async(item))
async def process_item_async(self, item: Any) -> Any:
- return await self._process_chain("process_item", item, self._spider)
+ return await self._process_chain("process_item", item, add_spider=True)
+
+ def _process_parallel(self, methodname: str) -> Deferred[list[None]]:
+ methods = cast("Iterable[Callable[..., None]]", self.methods[methodname])
+
+ def get_dfd(method: Callable[..., None]) -> Deferred[None]:
+ if method in self._mw_methods_requiring_spider:
+ return maybeDeferred_coro(method, self._spider)
+ return maybeDeferred_coro(method)
+
+ dfds = [get_dfd(m) for m in methods]
+ d: Deferred[list[tuple[bool, None]]] = DeferredList(
+ dfds, fireOnOneErrback=True, consumeErrors=True
+ )
+ d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
+
+ def eb(failure: Failure) -> Failure:
+ return failure.value.subFailure
+
+ d2.addErrback(eb)
+ return d2
+
+ def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
+ if spider:
+ self._warn_spider_arg("open_spider")
+ self._set_compat_spider(spider)
+ return self._process_parallel("open_spider")
+
+ def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
+ if spider:
+ self._warn_spider_arg("close_spider")
+ self._set_compat_spider(spider)
+ return self._process_parallel("close_spider")
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index a50150c59..fb65ff06a 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -165,12 +165,27 @@ class MediaPipeline(ABC):
pipe._finish_init(crawler)
return pipe
- def open_spider(self, spider: Spider) -> None:
- self.spiderinfo = self.SpiderInfo(spider)
+ def open_spider(self, spider: Spider | None = None) -> None:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to MediaPipeline.open_spider()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ assert self.crawler.spider
+ self.spiderinfo = self.SpiderInfo(self.crawler.spider)
def process_item(
- self, item: Any, spider: Spider
+ self, item: Any, spider: Spider | None = None
) -> Deferred[list[FileInfoOrError]]:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to MediaPipeline.process_item()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info, item) for r in requests]
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
index 196b84d0d..a047809c3 100644
--- a/scrapy/spidermiddlewares/base.py
+++ b/scrapy/spidermiddlewares/base.py
@@ -1,8 +1,10 @@
from __future__ import annotations
+import warnings
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
@@ -53,15 +55,32 @@ class BaseSpiderMiddleware:
yield o
def process_spider_output(
- self, response: Response, result: Iterable[Any], spider: Spider
+ self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o
async def process_spider_output_async(
- self, response: Response, result: AsyncIterator[Any], spider: Spider
+ self,
+ response: Response,
+ result: AsyncIterator[Any],
+ spider: Spider | None = None,
) -> AsyncIterator[Any]:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
async for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index fecdb0142..d23bca50c 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -7,8 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
+import warnings
from typing import TYPE_CHECKING, Any
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
if TYPE_CHECKING:
@@ -53,16 +55,33 @@ class DepthMiddleware(BaseSpiderMiddleware):
return o
def process_spider_output(
- self, response: Response, result: Iterable[Any], spider: Spider
+ self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to DepthMiddleware.process_spider_output()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self._init_depth(response)
- yield from super().process_spider_output(response, result, spider)
+ yield from super().process_spider_output(response, result)
async def process_spider_output_async(
- self, response: Response, result: AsyncIterator[Any], spider: Spider
+ self,
+ response: Response,
+ result: AsyncIterator[Any],
+ spider: Spider | None = None,
) -> AsyncIterator[Any]:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to DepthMiddleware.process_spider_output_async()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self._init_depth(response)
- async for o in super().process_spider_output_async(response, result, spider):
+ async for o in super().process_spider_output_async(response, result):
yield o
def _init_depth(self, response: Response) -> None:
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 42619ec7f..b4c9c10bf 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
+import warnings
from typing import TYPE_CHECKING, Any
-from scrapy.exceptions import IgnoreRequest
+from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -35,9 +36,7 @@ class HttpError(IgnoreRequest):
class HttpErrorMiddleware:
- @classmethod
- def from_crawler(cls, crawler: Crawler) -> Self:
- return cls(crawler.settings)
+ crawler: Crawler
def __init__(self, settings: BaseSettings):
self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL")
@@ -45,7 +44,22 @@ class HttpErrorMiddleware:
"HTTPERROR_ALLOWED_CODES"
)
- def process_spider_input(self, response: Response, spider: Spider) -> None:
+ @classmethod
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ o = cls(crawler.settings)
+ o.crawler = crawler
+ return o
+
+ def process_spider_input(
+ self, response: Response, spider: Spider | None = None
+ ) -> None:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to HttpErrorMiddleware.process_spider_input()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
if 200 <= response.status < 300: # common case
return
meta = response.meta
@@ -57,25 +71,34 @@ class HttpErrorMiddleware:
return
else:
allowed_statuses = getattr(
- spider, "handle_httpstatus_list", self.handle_httpstatus_list
+ self.crawler.spider,
+ "handle_httpstatus_list",
+ self.handle_httpstatus_list,
)
if response.status in allowed_statuses:
return
raise HttpError(response, "Ignoring non-200 response")
def process_spider_exception(
- self, response: Response, exception: Exception, spider: Spider
+ self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None:
+ if spider is not None: # pragma: no cover
+ warnings.warn(
+ "Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"
+ " is deprecated and the passed value is ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
if isinstance(exception, HttpError):
- assert spider.crawler.stats
- spider.crawler.stats.inc_value("httperror/response_ignored_count")
- spider.crawler.stats.inc_value(
+ assert self.crawler.stats
+ self.crawler.stats.inc_value("httperror/response_ignored_count")
+ self.crawler.stats.inc_value(
f"httperror/response_ignored_status_count/{response.status}"
)
logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
{"response": response},
- extra={"spider": spider},
+ extra={"spider": self.crawler.spider},
)
return []
return None
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index cbccbd244..7481e26ed 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -312,19 +312,6 @@ def process_chain(
return d
-async def _process_chain(
- callables: Iterable[Callable[Concatenate[_T, _P], _T | Awaitable[_T]]],
- input_: _T,
- *a: _P.args,
- **kw: _P.kwargs,
-) -> _T:
- """Chain the given (potentialy asynchronous) callables."""
- result = input_
- for callable_ in callables:
- result = await ensure_awaitable(callable_(result, *a, **kw))
- return result
-
-
def process_chain_both(
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
@@ -355,10 +342,15 @@ def process_parallel(
input: _T, # noqa: A002
*a: _P.args,
**kw: _P.kwargs,
-) -> Deferred[list[_T2]]:
+) -> Deferred[list[_T2]]: # pragma: no cover
"""Return a Deferred with the output of all successful calls to the given
callbacks
"""
+ warnings.warn(
+ "process_parallel() is deprecated.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks]
d: Deferred[list[tuple[bool, _T2]]] = DeferredList(
dfds, fireOnOneErrback=True, consumeErrors=True
diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
index c32aaf37d..e000a23db 100644
--- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
@@ -16,7 +16,7 @@ class UppercasePipeline:
def open_spider(self, spider):
return deferred_from_coro(self._open_spider(spider))
- def process_item(self, item, spider):
+ def process_item(self, item):
return {"url": item["url"].upper()}
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index 028e3a08a..eb2e1105c 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -16,7 +16,7 @@ class UppercasePipeline:
def open_spider(self, spider):
return deferred_from_coro(self._open_spider(spider))
- def process_item(self, item, spider):
+ def process_item(self, item):
return {"url": item["url"].upper()}
diff --git a/tests/pipelines.py b/tests/pipelines.py
index d9fc12676..4f00d6919 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -4,13 +4,13 @@ Some pipelines used for testing
class ZeroDivisionErrorPipeline:
- def open_spider(self, spider):
+ def open_spider(self):
1 / 0
- def process_item(self, item, spider):
+ def process_item(self, item):
return item
class ProcessWithZeroDivisionErrorPipeline:
- def process_item(self, item, spider):
+ def process_item(self, item):
1 / 0
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
index 3e29c70ed..577ecd686 100644
--- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
@@ -2,7 +2,7 @@ class TestSpiderPipeline:
def open_spider(self, spider):
pass
- def process_item(self, item, spider):
+ def process_item(self, item):
return item
@@ -10,5 +10,5 @@ class TestSpiderExceptionPipeline:
def open_spider(self, spider):
raise RuntimeError("exception")
- def process_item(self, item, spider):
+ def process_item(self, item):
return item
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index fe7a807d4..7b119d7cf 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -153,7 +153,7 @@ import logging
class MyPipeline:
component_name = 'my_pipeline'
- def process_item(self, item, spider):
+ def process_item(self, item):
logging.info('It Works!')
return item
""",
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index f21e52e52..426be13a7 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -246,7 +246,7 @@ class SkipMessagesLogFormatter(LogFormatter):
class DropSomeItemsPipeline:
drop = True
- def process_item(self, item, spider):
+ def process_item(self, item):
if self.drop:
self.drop = False
raise DropItem("Ignoring item")
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index b6a07f872..6c3008b25 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -7,7 +7,6 @@ import pytest
from scrapy import Spider
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
-from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@@ -57,7 +56,10 @@ class MyMiddlewareManager(MiddlewareManager):
return [M1, MOff, M3]
def _add_middleware(self, mw):
- super()._add_middleware(mw)
+ if hasattr(mw, "open_spider"):
+ self.methods["open_spider"].append(mw.open_spider)
+ if hasattr(mw, "close_spider"):
+ self.methods["close_spider"].appendleft(mw.close_spider)
if hasattr(mw, "process"):
self.methods["process"].append(mw.process)
@@ -105,73 +107,3 @@ def test_no_crawler() -> None:
mwman = MyMiddlewareManager(m1, m2, m3)
assert mwman.middlewares == (m1, m2, m3)
assert mwman.crawler is None
-
-
-def test_deprecated_spider_arg_no_crawler_spider(crawler: Crawler) -> None:
- """Crawler is provided, but doesn't have a spider. The instance passed to the method is
- ignored and raises a warning."""
- mwman = MyMiddlewareManager(crawler=crawler)
- with (
- pytest.warns(
- ScrapyDeprecationWarning,
- match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
- ),
- pytest.raises(
- ValueError,
- match="MyMiddlewareManager needs to access self.crawler.spider but it is None",
- ),
- ):
- mwman.open_spider(DefaultSpider())
-
-
-def test_deprecated_spider_arg_with_crawler(crawler: Crawler) -> None:
- """Crawler is provided and has a spider, works. The instance passed to the method is ignored,
- even if mismatched, but raises a warning."""
- mwman = MyMiddlewareManager(crawler=crawler)
- crawler.spider = crawler._create_spider("foo")
- with pytest.warns(
- ScrapyDeprecationWarning,
- match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
- ):
- mwman.open_spider(DefaultSpider())
- with pytest.warns(
- ScrapyDeprecationWarning,
- match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",
- ):
- mwman.close_spider(DefaultSpider())
-
-
-def test_deprecated_spider_arg_without_crawler() -> None:
- """The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""
- with pytest.warns(
- ScrapyDeprecationWarning,
- match="was called without the crawler argument",
- ):
- mwman = MyMiddlewareManager()
- with pytest.warns(
- ScrapyDeprecationWarning,
- match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
- ):
- mwman.open_spider(DefaultSpider())
- with (
- pytest.warns(
- ScrapyDeprecationWarning,
- match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",
- ),
- pytest.raises(RuntimeError, match="Different instances of Spider were passed"),
- ):
- mwman.close_spider(DefaultSpider())
-
-
-def test_no_spider_arg_without_crawler() -> None:
- """If no crawler and no spider arg, raise an error."""
- with pytest.warns(
- ScrapyDeprecationWarning,
- match="was called without the crawler argument",
- ):
- mwman = MyMiddlewareManager()
- with pytest.raises(
- ValueError,
- match="has no known Spider instance",
- ):
- mwman.open_spider()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index a4cdaa5f8..db64589af 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -29,6 +29,7 @@ from scrapy.pipelines.files import (
GCSFilesStore,
S3FilesStore,
)
+from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
@@ -78,10 +79,11 @@ class TestFilesPipeline:
def setup_method(self):
self.tempdir = mkdtemp()
settings_dict = {"FILES_STORE": self.tempdir}
- crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+ crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
+ crawler.spider = crawler._create_spider()
self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func
- self.pipeline.open_spider(None)
+ self.pipeline.open_spider()
def teardown_method(self):
rmtree(self.tempdir)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 42ae897e8..cc1d73349 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -13,10 +13,10 @@ from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline
-from scrapy.spiders import Spider
from scrapy.utils.asyncio import call_later
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
+from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@@ -48,12 +48,11 @@ class TestBaseMediaPipeline:
settings = None
def setup_method(self):
- spider_cls = Spider
- self.spider = spider_cls("media.com")
- crawler = get_crawler(spider_cls, self.settings)
+ crawler = get_crawler(DefaultSpider, self.settings)
+ crawler.spider = crawler._create_spider()
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
- self.pipe.open_spider(self.spider)
+ self.pipe.open_spider()
self.info = self.pipe.spiderinfo
self.fingerprint = crawler.request_fingerprinter.fingerprint
@@ -162,7 +161,7 @@ class TestBaseMediaPipeline:
@inlineCallbacks
def test_default_process_item(self):
item = {"name": "name"}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item is item
@@ -216,7 +215,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
errback=self._errback,
)
item = {"requests": req}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, {})]
assert self.pipe._mockcalled == [
"get_media_requests",
@@ -236,7 +235,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
errback=self._errback,
)
item = {"requests": req}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 1
assert new_item["results"][0][0] is False
assert isinstance(new_item["results"][0][1], Failure)
@@ -258,7 +257,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
fail = Failure(exc)
req2 = Request("http://url2", meta={"response": fail})
item = {"requests": [req1, req2]}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 2
assert new_item["results"][0] == (True, {})
assert new_item["results"][1][0] is False
@@ -281,7 +280,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
# returns single Request (without callback)
req = Request("http://url")
item = {"requests": req} # pass a single item
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item is item
assert self.fingerprint(req) in self.info.downloaded
@@ -289,7 +288,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1 = Request("http://url1")
req2 = Request("http://url2")
item = {"requests": iter([req1, req2])}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item is item
assert self.fingerprint(req1) in self.info.downloaded
assert self.fingerprint(req2) in self.info.downloaded
@@ -299,7 +298,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1})
item = {"requests": req1}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item is item
assert new_item["results"] == [(True, {})]
@@ -308,7 +307,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1.url, meta={"response": Response("http://donot.download.me")}
)
item = {"requests": req2}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item is item
assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})]
@@ -321,7 +320,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1.url, meta={"response": Response("http://donot.download.me")}
)
item = {"requests": [req1, req2]}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item is item
assert new_item["results"] == [(True, {}), (True, {})]
@@ -348,14 +347,14 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1 = Request("http://url", meta={"response": rsp1_func})
req2 = Request(req1.url, meta={"response": rsp2_func})
item = {"requests": [req1, req2]}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, {}), (True, {})]
@inlineCallbacks
def test_use_media_to_download_result(self):
req = Request("http://url", meta={"result": "ITSME"})
item = {"requests": req}
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, "ITSME")]
assert self.pipe._mockcalled == [
"get_media_requests",
@@ -555,12 +554,11 @@ class TestMediaFailedFailure:
settings = None
def setup_method(self):
- spider_cls = Spider
- self.spider = spider_cls("media.com")
- crawler = get_crawler(spider_cls, self.settings)
+ crawler = get_crawler(DefaultSpider, self.settings)
+ crawler.spider = crawler._create_spider()
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
- self.pipe.open_spider(self.spider)
+ self.pipe.open_spider()
self.info = self.pipe.spiderinfo
self.fingerprint = crawler.request_fingerprinter.fingerprint
@@ -587,7 +585,7 @@ class TestMediaFailedFailure:
with pytest.warns(
ScrapyDeprecationWarning, match="media_failed returned a Failure instance"
):
- new_item = yield self.pipe.process_item(item, self.spider)
+ new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 1
assert new_item["results"][0][0] is False
assert isinstance(new_item["results"][0][1], Failure)
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 725ceccee..4b8007ead 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -4,6 +4,7 @@ import pytest
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
@@ -19,17 +20,25 @@ from tests.mockserver.http import MockServer
class SimplePipeline:
- def process_item(self, item, spider):
+ def process_item(self, item):
item["pipeline_passed"] = True
return item
+class DeprecatedSpiderArgPipeline:
+ def open_spider(self, spider):
+ pass
+
+ def close_spider(self, spider):
+ pass
+
+
class DeferredPipeline:
def cb(self, item):
item["pipeline_passed"] = True
return item
- def process_item(self, item, spider):
+ def process_item(self, item):
d = Deferred()
d.addCallback(self.cb)
d.callback(item)
@@ -37,7 +46,7 @@ class DeferredPipeline:
class AsyncDefPipeline:
- async def process_item(self, item, spider):
+ async def process_item(self, item):
d = Deferred()
call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
@@ -46,7 +55,7 @@ class AsyncDefPipeline:
class AsyncDefAsyncioPipeline:
- async def process_item(self, item, spider):
+ async def process_item(self, item):
d = Deferred()
loop = asyncio.get_event_loop()
loop.call_later(0, d.callback, None)
@@ -57,7 +66,7 @@ class AsyncDefAsyncioPipeline:
class AsyncDefNotAsyncioPipeline:
- async def process_item(self, item, spider):
+ async def process_item(self, item):
d1 = Deferred()
from twisted.internet import reactor
@@ -184,6 +193,9 @@ class TestCustomPipelineManager:
def open_spider(self, spider): # pylint: disable=signature-differs
return super().open_spider(spider)
+ def close_spider(self, spider): # pylint: disable=signature-differs
+ return super().close_spider(spider)
+
def process_item(self, item, spider): # pylint: disable=signature-differs
with pytest.warns(
ScrapyDeprecationWarning,
@@ -212,6 +224,10 @@ class TestCustomPipelineManager:
ScrapyDeprecationWarning,
match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",
),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",
+ ),
pytest.warns(
ScrapyDeprecationWarning,
match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument",
@@ -220,6 +236,10 @@ class TestCustomPipelineManager:
ScrapyDeprecationWarning,
match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated",
),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to CustomPipelineManager.close_spider\(\) is deprecated",
+ ),
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",
@@ -254,7 +274,7 @@ class TestCustomPipelineManager:
def process_item(self, item, spider):
for pipeline in self.pipelines:
- item = pipeline.process_item(item, spider)
+ item = pipeline.process_item(item)
return succeed(item)
items = []
@@ -294,3 +314,128 @@ class TestCustomPipelineManager:
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
assert len(items) == 1
+
+
+class TestMiddlewareManagerSpider:
+ """Tests for the deprecated spider arg handling in MiddlewareManager.
+
+ Here because MiddlewareManager doesn't have methods that could take a spider arg."""
+
+ @pytest.fixture
+ def crawler(self) -> Crawler:
+ return get_crawler(Spider)
+
+ def test_deprecated_spider_arg_no_crawler_spider(self, crawler: Crawler) -> None:
+ """Crawler is provided, but doesn't have a spider. The instance passed to the method is
+ ignored and raises a warning."""
+ mwman = ItemPipelineManager(crawler=crawler)
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
+ ),
+ ):
+ mwman._add_middleware(DeprecatedSpiderArgPipeline())
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
+ ),
+ pytest.raises(
+ ValueError,
+ match="ItemPipelineManager needs to access self.crawler.spider but it is None",
+ ),
+ ):
+ mwman.open_spider(DefaultSpider())
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
+ ),
+ pytest.raises(
+ ValueError,
+ match="ItemPipelineManager needs to access self.crawler.spider but it is None",
+ ),
+ ):
+ mwman.close_spider(DefaultSpider())
+
+ def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:
+ """Crawler is provided and has a spider, works. The instance passed to the method is ignored,
+ even if mismatched, but raises a warning."""
+ mwman = ItemPipelineManager(crawler=crawler)
+ crawler.spider = crawler._create_spider("foo")
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
+ ):
+ mwman.open_spider(DefaultSpider())
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
+ ):
+ mwman.close_spider(DefaultSpider())
+
+ def test_deprecated_spider_arg_without_crawler(self) -> None:
+ """The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="was called without the crawler argument",
+ ):
+ mwman = ItemPipelineManager()
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
+ ),
+ ):
+ mwman._add_middleware(DeprecatedSpiderArgPipeline())
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
+ ):
+ mwman.open_spider(DefaultSpider())
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
+ ),
+ pytest.raises(
+ RuntimeError, match="Different instances of Spider were passed"
+ ),
+ ):
+ mwman.close_spider(DefaultSpider())
+ mwman.close_spider()
+
+ def test_no_spider_arg_without_crawler(self) -> None:
+ """If no crawler and no spider arg, raise an error."""
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="was called without the crawler argument",
+ ):
+ mwman = ItemPipelineManager()
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
+ ),
+ ):
+ mwman._add_middleware(DeprecatedSpiderArgPipeline())
+ with (
+ pytest.raises(
+ ValueError,
+ match="has no known Spider instance",
+ ),
+ ):
+ mwman.open_spider()
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 041fc9588..0f31af8ad 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -33,12 +33,12 @@ class InjectArgumentsSpiderMiddleware:
request.cb_kwargs["from_process_start"] = True
yield request
- def process_spider_input(self, response, spider):
+ def process_spider_input(self, response):
request = response.request
if request.callback.__name__ == "parse_spider_mw":
request.cb_kwargs["from_process_spider_input"] = True
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
for element in result:
if (
isinstance(element, Request)
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 76609ac52..c1f358bd2 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -451,7 +451,7 @@ class TestSettings:
def test_passing_objects_as_values(self):
class TestPipeline:
- def process_item(self, i, s):
+ def process_item(self, i):
return i
settings = Settings(
@@ -471,7 +471,7 @@ class TestSettings:
assert priority == 800
assert mypipeline == TestPipeline
assert isinstance(mypipeline(), TestPipeline)
- assert mypipeline().process_item("item", None) == "item"
+ assert mypipeline().process_item("item") == "item"
myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
assert myhandler == FileDownloadHandler
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2ef810852..8fd33eb13 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -10,7 +10,7 @@ from testfixtures import LogCapture
from twisted.internet import defer
from scrapy.core.spidermw import SpiderMiddlewareManager
-from scrapy.exceptions import _InvalidOutput
+from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
@@ -54,7 +54,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware:
- def process_spider_input(self, response, spider):
+ def process_spider_input(self, response):
return 1
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
@@ -68,7 +68,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
return 1
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
@@ -82,11 +82,11 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware:
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
return 1
class RaiseExceptionProcessSpiderOutputMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
raise RuntimeError
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
@@ -101,11 +101,11 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware:
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
return None
class RaiseExceptionProcessSpiderOutputMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
1 / 0
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
@@ -191,34 +191,34 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
class ProcessSpiderOutputSimpleMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
yield from result
class ProcessSpiderOutputAsyncGenMiddleware:
- async def process_spider_output(self, response, result, spider):
+ async def process_spider_output(self, response, result):
async for r in result:
yield r
class ProcessSpiderOutputUniversalMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
yield from result
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
async for r in result:
yield r
class ProcessSpiderExceptionSimpleIterableMiddleware:
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
yield {"foo": 1}
yield {"foo": 2}
yield {"foo": 3}
class ProcessSpiderExceptionAsyncIteratorMiddleware:
- async def process_spider_exception(self, response, exception, spider):
+ async def process_spider_exception(self, response, exception):
yield {"foo": 1}
d = defer.Deferred()
call_later(0, d.callback, None)
@@ -315,12 +315,12 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
class ProcessSpiderOutputNonIterableMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
return
class ProcessSpiderOutputCoroutineMiddleware:
- async def process_spider_output(self, response, result, spider):
+ async def process_spider_output(self, response, result):
return result
@@ -384,23 +384,23 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
class UniversalMiddlewareNoSync:
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
yield
class UniversalMiddlewareBothSync:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
yield
- def process_spider_output_async(self, response, result, spider):
+ def process_spider_output_async(self, response, result):
yield
class UniversalMiddlewareBothAsync:
- async def process_spider_output(self, response, result, spider):
+ async def process_spider_output(self, response, result):
yield
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
yield
@@ -600,3 +600,34 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
async def test_exc_async_simple(self):
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
+
+
+class TestDeprecatedSpiderArg(TestSpiderMiddleware):
+ @deferred_f_from_coro_f
+ async def test_deprecated_spider_arg(self):
+ class DeprecatedSpiderArgMiddleware:
+ def process_spider_input(self, response, spider):
+ return None
+
+ def process_spider_output(self, response, result, spider):
+ 1 / 0
+
+ def process_spider_exception(self, response, exception, spider):
+ return []
+
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"process_spider_input\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"process_spider_output\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"process_spider_exception\(\) requires a spider argument",
+ ),
+ ):
+ self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
+ await self._scrape_response()
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 03446db7e..00e547305 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -21,12 +21,6 @@ def crawler() -> Crawler:
return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})
-def spider(crawler: Crawler) -> Spider:
- crawler.spider = crawler._create_spider("scrapytest.org")
- return crawler.spider
-
-
@pytest.fixture
def stats(crawler: Crawler) -> Generator[StatsCollector]:
assert crawler.stats is not None
@@ -42,15 +36,13 @@ def mw(crawler: Crawler) -> DepthMiddleware:
return DepthMiddleware.from_crawler(crawler)
-def test_process_spider_output(
- mw: DepthMiddleware, stats: StatsCollector, spider: Spider
-) -> None:
+def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:
req = Request("http://scrapytest.org")
resp = Response("http://scrapytest.org")
resp.request = req
result = [Request("http://scrapytest.org")]
- out = list(mw.process_spider_output(resp, result, spider))
+ out = list(mw.process_spider_output(resp, result))
assert out == result
rdc = stats.get_value("request_depth_count/1")
@@ -58,7 +50,7 @@ def test_process_spider_output(
req.meta["depth"] = 1
- out2 = list(mw.process_spider_output(resp, result, spider))
+ out2 = list(mw.process_spider_output(resp, result))
assert not out2
rdm = stats.get_value("request_depth_max")
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index b41ae986e..9899a0cd7 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -7,9 +7,8 @@ from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks
from scrapy.http import Request, Response
-from scrapy.settings import Settings
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
-from scrapy.spiders import Spider
+from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
@@ -57,12 +56,6 @@ def _response(request: Request, status_code: int) -> Response:
return Response(request.url, status=status_code, request=request)
-def spider() -> Spider:
- crawler = get_crawler(Spider)
- return Spider.from_crawler(crawler, name="foo")
-
-
@pytest.fixture
def res200() -> Response:
return _response(req, 200)
@@ -81,36 +74,35 @@ def res404() -> Response:
class TestHttpErrorMiddleware:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
- return HttpErrorMiddleware(Settings({}))
+ crawler = get_crawler(DefaultSpider)
+ crawler.spider = crawler._create_spider()
+ return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input(
- self,
- mw: HttpErrorMiddleware,
- spider: Spider,
- res200: Response,
- res404: Response,
+ self, mw: HttpErrorMiddleware, res200: Response, res404: Response
) -> None:
- mw.process_spider_input(res200, spider)
+ mw.process_spider_input(res200)
with pytest.raises(HttpError):
- mw.process_spider_input(res404, spider)
+ mw.process_spider_input(res404)
def test_process_spider_exception(
- self, mw: HttpErrorMiddleware, spider: Spider, res404: Response
+ self, mw: HttpErrorMiddleware, res404: Response
) -> None:
- assert mw.process_spider_exception(res404, HttpError(res404), spider) == []
- assert mw.process_spider_exception(res404, Exception(), spider) is None
+ assert mw.process_spider_exception(res404, HttpError(res404)) == []
+ assert mw.process_spider_exception(res404, Exception()) is None
def test_handle_httpstatus_list(
- self, mw: HttpErrorMiddleware, spider: Spider, res404: Response
+ self, mw: HttpErrorMiddleware, res404: Response
) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
res = _response(request, 404)
- mw.process_spider_input(res, spider)
+ mw.process_spider_input(res)
- spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
- mw.process_spider_input(res404, spider)
+ assert mw.crawler.spider
+ mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
+ mw.process_spider_input(res404)
class TestHttpErrorMiddlewareSettings:
@@ -118,76 +110,71 @@ class TestHttpErrorMiddlewareSettings:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
- return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
+ crawler.spider = crawler._create_spider()
+ return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input(
self,
mw: HttpErrorMiddleware,
- spider: Spider,
res200: Response,
res402: Response,
res404: Response,
) -> None:
- mw.process_spider_input(res200, spider)
+ mw.process_spider_input(res200)
with pytest.raises(HttpError):
- mw.process_spider_input(res404, spider)
- mw.process_spider_input(res402, spider)
+ mw.process_spider_input(res404)
+ mw.process_spider_input(res402)
- def test_meta_overrides_settings(
- self, mw: HttpErrorMiddleware, spider: Spider
- ) -> None:
+ def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
res404 = _response(request, 404)
res402 = _response(request, 402)
- mw.process_spider_input(res404, spider)
+ mw.process_spider_input(res404)
with pytest.raises(HttpError):
- mw.process_spider_input(res402, spider)
+ mw.process_spider_input(res402)
def test_spider_override_settings(
- self,
- mw: HttpErrorMiddleware,
- spider: Spider,
- res402: Response,
- res404: Response,
+ self, mw: HttpErrorMiddleware, res402: Response, res404: Response
) -> None:
- spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
- mw.process_spider_input(res404, spider)
+ assert mw.crawler.spider
+ mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
+ mw.process_spider_input(res404)
with pytest.raises(HttpError):
- mw.process_spider_input(res402, spider)
+ mw.process_spider_input(res402)
class TestHttpErrorMiddlewareHandleAll:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
- return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
+ crawler.spider = crawler._create_spider()
+ return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input(
self,
mw: HttpErrorMiddleware,
- spider: Spider,
res200: Response,
res404: Response,
) -> None:
- mw.process_spider_input(res200, spider)
- mw.process_spider_input(res404, spider)
+ mw.process_spider_input(res200)
+ mw.process_spider_input(res404)
- def test_meta_overrides_settings(
- self, mw: HttpErrorMiddleware, spider: Spider
- ) -> None:
+ def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
res404 = _response(request, 404)
res402 = _response(request, 402)
- mw.process_spider_input(res404, spider)
+ mw.process_spider_input(res404)
with pytest.raises(HttpError):
- mw.process_spider_input(res402, spider)
+ mw.process_spider_input(res402)
- def test_httperror_allow_all_false(self, spider: Spider) -> None:
+ def test_httperror_allow_all_false(self) -> None:
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
request_httpstatus_false = Request(
@@ -200,8 +187,8 @@ class TestHttpErrorMiddlewareHandleAll:
res402 = _response(request_httpstatus_true, 402)
with pytest.raises(HttpError):
- mw.process_spider_input(res404, spider)
- mw.process_spider_input(res402, spider)
+ mw.process_spider_input(res404)
+ mw.process_spider_input(res402)
class TestHttpErrorMiddlewareIntegrational:
@@ -218,7 +205,7 @@ class TestHttpErrorMiddlewareIntegrational:
def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
- assert not crawler.spider.skipped, crawler.spider.skipped
+ assert not crawler.spider.skipped
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 3f2b2057b..1808e087c 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -6,18 +6,27 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
-class LogExceptionMiddleware:
- def process_spider_exception(self, response, exception, spider):
- spider.logger.info(
+class _BaseSpiderMiddleware:
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+
+class LogExceptionMiddleware(_BaseSpiderMiddleware):
+ def process_spider_exception(self, response, exception):
+ self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__
)
# ================================================================================
# (0) recover from an exception on a spider callback
-class RecoveryMiddleware:
- def process_spider_exception(self, response, exception, spider):
- spider.logger.info(
+class RecoveryMiddleware(_BaseSpiderMiddleware):
+ def process_spider_exception(self, response, exception):
+ self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__
)
return [
@@ -55,9 +64,9 @@ class RecoveryAsyncGenSpider(RecoverySpider):
# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
-class FailProcessSpiderInputMiddleware:
- def process_spider_input(self, response, spider):
- spider.logger.info("Middleware: will raise IndexError")
+class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
+ def process_spider_input(self, response):
+ self.crawler.spider.logger.info("Middleware: will raise IndexError")
raise IndexError
@@ -159,27 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
# ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator)
-class _GeneratorDoNothingMiddleware:
- def process_spider_output(self, response, result, spider):
+class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
-class GeneratorFailMiddleware:
- def process_spider_output(self, response, result, spider):
+class GeneratorFailMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
raise LookupError
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
yield {"processed": [method]}
@@ -187,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
pass
-class GeneratorRecoverMiddleware:
- def process_spider_output(self, response, result, spider):
+class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
yield {"processed": [method]}
@@ -226,30 +241,34 @@ class GeneratorOutputChainSpider(Spider):
# (5) exceptions from a middleware process_spider_output method (not generator)
-class _NotGeneratorDoNothingMiddleware:
- def process_spider_output(self, response, result, spider):
+class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
-class NotGeneratorFailMiddleware:
- def process_spider_output(self, response, result, spider):
+class NotGeneratorFailMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
raise ReferenceError
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
return [{"processed": [method]}]
@@ -257,17 +276,19 @@ class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddlewa
pass
-class NotGeneratorRecoverMiddleware:
- def process_spider_output(self, response, result, spider):
+class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
return [{"processed": [method]}]
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 895df1804..658351999 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -61,11 +61,11 @@ class TestRefererMiddleware:
def get_response(self, origin: str) -> Response:
return Response(origin, headers=self.resp_headers)
- def test(self, mw: RefererMiddleware, spider: Spider) -> None:
+ def test(self, mw: RefererMiddleware) -> None:
for origin, target, referrer in self.scenarii:
response = self.get_response(origin)
request = self.get_request(target)
- out = list(mw.process_spider_output(response, [request], spider))
+ out = list(mw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == referrer
@@ -1036,7 +1036,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
response = self.get_response(parent)
request = self.get_request(target)
- out = list(referrermw.process_spider_output(response, [request], spider))
+ out = list(referrermw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == init_referrer
for status, url in redirections:
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index ca65529a3..750ae3b07 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -27,11 +27,6 @@ def crawler() -> Crawler:
return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength})
-def spider(crawler: Crawler) -> Spider:
- return crawler._create_spider("foo")
-
-
@pytest.fixture
def stats(crawler: Crawler) -> StatsCollector:
assert crawler.stats is not None
@@ -43,22 +38,19 @@ def mw(crawler: Crawler) -> UrlLengthMiddleware:
return UrlLengthMiddleware.from_crawler(crawler)
-def process_spider_output(mw: UrlLengthMiddleware, spider: Spider) -> list[Request]:
- return list(mw.process_spider_output(response, reqs, spider))
+def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:
+ return list(mw.process_spider_output(response, reqs))
-def test_middleware_works(mw: UrlLengthMiddleware, spider: Spider) -> None:
- assert process_spider_output(mw, spider) == [short_url_req]
+def test_middleware_works(mw: UrlLengthMiddleware) -> None:
+ assert process_spider_output(mw) == [short_url_req]
def test_logging(
- stats: StatsCollector,
- mw: UrlLengthMiddleware,
- spider: Spider,
- caplog: pytest.LogCaptureFixture,
+ stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(INFO):
- process_spider_output(mw, spider)
+ process_spider_output(mw)
ric = stats.get_value("urllength/request_ignored_count")
assert ric == 1
assert f"Ignoring link (url length > {maxlength})" in caplog.text
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 7570fb96c..df7f1a20e 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
- _process_chain,
aiter_errback,
deferred_f_from_coro_f,
deferred_from_coro,
@@ -19,7 +18,6 @@ from scrapy.utils.defer import (
maybe_deferred_to_future,
mustbe_deferred,
parallel_async,
- process_parallel,
)
if TYPE_CHECKING:
@@ -85,26 +83,6 @@ def eb1(failure, arg1, arg2):
return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"
-class TestDeferUtils:
- @deferred_f_from_coro_f
- async def test_process_chain(self):
- x = await _process_chain([cb1, cb2, cb3], "res", "v1", "v2")
- assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)"
-
- with pytest.raises(TypeError):
- await _process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")
-
- @inlineCallbacks
- def test_process_parallel(self):
- x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
- assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]
-
- @inlineCallbacks
- def test_process_parallel_failure(self):
- with pytest.raises(TypeError):
- yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
-
-
class TestIterErrback:
def test_iter_errback_good(self):
def itergood() -> Generator[int, None, None]: