Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Extract @_warn_spider_arg. (#7033)

Andrey Rakhmatullinc097921c4475
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 3c7960b86..4b4d4efc7 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -21,6 +21,7 @@ from scrapy.utils.asyncio import (
call_later,
create_looping_call,
)
+from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
_defer_sleep_async,
_schedule_coro,
@@ -144,15 +145,10 @@ class Downloader:
)
@inlineCallbacks
+ @_warn_spider_arg
def fetch(
self, request: Request, spider: Spider | None = None
) -> Generator[Deferred[Any], Any, Response | Request]:
- if spider is not None:
- warnings.warn(
- "Passing a 'spider' argument to Downloader.fetch() is deprecated.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
self.active.add(request)
try:
return (yield self.middleware.download(self._enqueue_request, request))
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index fe3c780e3..6352ede3c 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -3,13 +3,13 @@
from __future__ import annotations
import logging
-import warnings
from typing import TYPE_CHECKING, Any, Protocol, cast
from twisted.internet import defer
from scrapy import Request, Spider, signals
-from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
@@ -94,15 +94,10 @@ class DownloadHandlers:
self._handlers[scheme] = dh
return dh
+ @_warn_spider_arg
def download_request(
self, request: Request, spider: Spider | None = None
) -> Deferred[Response]:
- if spider is not None:
- warnings.warn(
- "Passing a 'spider' argument to DownloadHandlers.download_request() is deprecated.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 769b32fc9..9184731cd 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -23,6 +23,7 @@ from scrapy.exceptions import (
from scrapy.http import Request, Response
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
+from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
_defer_sleep_async,
_schedule_coro,
@@ -219,16 +220,10 @@ class Scraper:
self.slot.closing.callback(self.crawler.spider)
@inlineCallbacks
+ @_warn_spider_arg
def enqueue_scrape(
self, result: Response | Failure, request: Request, spider: Spider | None = None
) -> Generator[Deferred[Any], Any, None]:
- if spider is not None:
- warnings.warn(
- "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
if self.slot is None:
raise RuntimeError("Scraper slot not assigned")
dfd = self.slot.add_response_request(result, request)
@@ -343,6 +338,7 @@ class Scraper:
# which needs to be passed to iterate_spider_output()
return await ensure_awaitable(iterate_spider_output(output))
+ @_warn_spider_arg
def handle_spider_error(
self,
_failure: Failure,
@@ -351,13 +347,6 @@ class Scraper:
spider: Spider | None = None,
) -> None:
"""Handle an exception raised by a spider callback or errback."""
- if spider is not None:
- warnings.warn(
- "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
assert self.crawler.spider
exc = _failure.value
if isinstance(exc, CloseSpider):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index fb65ff06a..a7cd51166 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -22,6 +22,7 @@ from scrapy.http.request import NO_CALLBACK, Request
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.datatypes import SequenceExclude
+from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
@@ -165,27 +166,15 @@ class MediaPipeline(ABC):
pipe._finish_init(crawler)
return pipe
+ @_warn_spider_arg
def open_spider(self, spider: Spider | None = None) -> None:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to MediaPipeline.open_spider()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
assert self.crawler.spider
self.spiderinfo = self.SpiderInfo(self.crawler.spider)
+ @_warn_spider_arg
def process_item(
self, item: Any, spider: Spider | None = None
) -> Deferred[list[FileInfoOrError]]:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to MediaPipeline.process_item()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info, item) for r in requests]
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
index a047809c3..889fc6df1 100644
--- a/scrapy/spidermiddlewares/base.py
+++ b/scrapy/spidermiddlewares/base.py
@@ -1,10 +1,9 @@
from __future__ import annotations
-import warnings
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
@@ -54,33 +53,21 @@ class BaseSpiderMiddleware:
if (o := self._get_processed(o, None)) is not None:
yield o
+ @_warn_spider_arg
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o
+ @_warn_spider_arg
async def process_spider_output_async(
self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
async for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index d23bca50c..9c387530c 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -7,11 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
-import warnings
from typing import TYPE_CHECKING, Any
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
@@ -54,32 +53,20 @@ class DepthMiddleware(BaseSpiderMiddleware):
o.crawler = crawler
return o
+ @_warn_spider_arg
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to DepthMiddleware.process_spider_output()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
self._init_depth(response)
yield from super().process_spider_output(response, result)
+ @_warn_spider_arg
async def process_spider_output_async(
self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to DepthMiddleware.process_spider_output_async()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
self._init_depth(response)
async for o in super().process_spider_output_async(response, result):
yield o
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index b4c9c10bf..c160d1adb 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -7,10 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
-import warnings
from typing import TYPE_CHECKING, Any
-from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -50,16 +50,10 @@ class HttpErrorMiddleware:
o.crawler = crawler
return o
+ @_warn_spider_arg
def process_spider_input(
self, response: Response, spider: Spider | None = None
) -> None:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to HttpErrorMiddleware.process_spider_input()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
if 200 <= response.status < 300: # common case
return
meta = response.meta
@@ -79,16 +73,10 @@ class HttpErrorMiddleware:
return
raise HttpError(response, "Ignoring non-200 response")
+ @_warn_spider_arg
def process_spider_exception(
self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None:
- if spider is not None: # pragma: no cover
- warnings.warn(
- "Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"
- " is deprecated and the passed value is ignored.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
if isinstance(exception, HttpError):
assert self.crawler.stats
self.crawler.stats.inc_value("httperror/response_ignored_count")
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index c2fa0ae6a..02adef1e4 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -4,13 +4,11 @@ Scrapy extension for collecting scraping stats
from __future__ import annotations
-import inspect
import logging
import pprint
-import warnings
from typing import TYPE_CHECKING, Any
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from scrapy import Spider
@@ -44,22 +42,7 @@ class StatsCollector:
"open_spider",
"close_spider",
) and callable(original_attr):
-
- def _deprecated_wrapper(*args, **kwargs):
- sig = inspect.signature(original_attr).bind(*args, **kwargs)
- sig.apply_defaults()
-
- if sig.arguments.get("spider"):
- warnings.warn(
- f"Passing a 'spider' argument to StatsCollector.{name}() is deprecated and"
- f" the argument will be removed in a future Scrapy version.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
- return original_attr(*args, **kwargs)
-
- return _deprecated_wrapper
+ return _warn_spider_arg(original_attr)
return original_attr
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 0f4d0beda..c4739bcee 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -1,8 +1,9 @@
from __future__ import annotations
+import inspect
import warnings
from functools import wraps
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar, overload
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.threads import deferToThread
@@ -10,7 +11,7 @@ from twisted.internet.threads import deferToThread
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import AsyncGenerator, Callable, Coroutine
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
@@ -30,7 +31,7 @@ def deprecated(
def deco(func: Callable[_P, _T]) -> Callable[_P, _T]:
@wraps(func)
- def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any:
+ def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T:
message = f"Call to deprecated function {func.__name__}."
if use_instead:
message += f" Use {use_instead} instead."
@@ -65,3 +66,67 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
return deferToThread(func, *a, **kw)
return wrapped
+
+
+@overload
+def _warn_spider_arg(
+ func: Callable[_P, Coroutine[Any, Any, _T]],
+) -> Callable[_P, Coroutine[Any, Any, _T]]: ...
+
+
+@overload
+def _warn_spider_arg(
+ func: Callable[_P, AsyncGenerator[_T]],
+) -> Callable[_P, AsyncGenerator[_T]]: ...
+
+
+@overload
+def _warn_spider_arg(func: Callable[_P, _T]) -> Callable[_P, _T]: ...
+
+
+def _warn_spider_arg(
+ func: Callable[_P, _T],
+) -> (
+ Callable[_P, _T]
+ | Callable[_P, Coroutine[Any, Any, _T]]
+ | Callable[_P, AsyncGenerator[_T]]
+):
+ """Decorator to warn if a ``spider`` argument is passed to a function."""
+
+ def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None:
+ bound = inspect.signature(func).bind(*args, **kwargs)
+ if "spider" in bound.arguments:
+ warnings.warn(
+ f"Passing a 'spider' argument to {func.__qualname__}() is deprecated and "
+ "the argument will be removed in a future Scrapy version.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=3,
+ )
+
+ if inspect.iscoroutinefunction(func):
+
+ @wraps(func)
+ async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+ check_args(*args, **kwargs)
+ return await func(*args, **kwargs)
+
+ return async_inner
+
+ if inspect.isasyncgenfunction(func):
+
+ @wraps(func)
+ async def asyncgen_inner(
+ *args: _P.args, **kwargs: _P.kwargs
+ ) -> AsyncGenerator[_T]:
+ check_args(*args, **kwargs)
+ async for item in func(*args, **kwargs):
+ yield item
+
+ return asyncgen_inner
+
+ @wraps(func)
+ def sync_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+ check_args(*args, **kwargs)
+ return func(*args, **kwargs)
+
+ return sync_inner
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 5276636fc..99d5cbb6d 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -180,7 +180,7 @@ class TestFilesPipeline:
for p in patchers:
p.start()
- result = yield self.pipeline.process_item(item, None)
+ result = yield self.pipeline.process_item(item)
assert result["files"][0]["checksum"] == "abc"
assert result["files"][0]["status"] == "uptodate"
@@ -211,7 +211,7 @@ class TestFilesPipeline:
for p in patchers:
p.start()
- result = yield self.pipeline.process_item(item, None)
+ result = yield self.pipeline.process_item(item)
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "downloaded"
@@ -242,7 +242,7 @@ class TestFilesPipeline:
for p in patchers:
p.start()
- result = yield self.pipeline.process_item(item, None)
+ result = yield self.pipeline.process_item(item)
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "cached"
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
index e2ee48e1d..da570904d 100644
--- a/tests/test_spidermiddleware_base.py
+++ b/tests/test_spidermiddleware_base.py
@@ -28,7 +28,7 @@ def test_trivial(crawler: Crawler) -> None:
test_req = Request("data:,")
spider_output = [test_req, {"foo": "bar"}]
for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert processed == [test_req, {"foo": "bar"}]
@@ -51,7 +51,7 @@ def test_processed_request(crawler: Crawler) -> None:
test_req3 = Request("data:3,")
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert len(processed) == 3
@@ -75,7 +75,7 @@ def test_processed_item(crawler: Crawler) -> None:
test_req = Request("data:,")
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert processed == [{"foo": 1}, test_req, {"foo": 30}]
@@ -112,7 +112,7 @@ def test_processed_both(crawler: Crawler) -> None:
test_req3,
]
for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert len(processed) == 4