Scrapy, a fast high-level web crawling & scraping framework for Python.
Extract @_warn_spider_arg. (#7033)
Andrey Rakhmatullinc097921c4475
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 3c7960b86..4b4d4efc7 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -21,6 +21,7 @@ from scrapy.utils.asyncio import (call_later,create_looping_call,)+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (_defer_sleep_async,_schedule_coro,@@ -144,15 +145,10 @@ class Downloader:)@inlineCallbacks+ @_warn_spider_argdef fetch(self, request: Request, spider: Spider | None = None) -> Generator[Deferred[Any], Any, Response | Request]:- if spider is not None:- warnings.warn(- "Passing a 'spider' argument to Downloader.fetch() is deprecated.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )self.active.add(request)try:return (yield self.middleware.download(self._enqueue_request, request))diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex fe3c780e3..6352ede3c 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -3,13 +3,13 @@from __future__ import annotationsimport logging-import warningsfrom typing import TYPE_CHECKING, Any, Protocol, castfrom twisted.internet import deferfrom scrapy import Request, Spider, signals-from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning+from scrapy.exceptions import NotConfigured, NotSupported+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import without_none_values@@ -94,15 +94,10 @@ class DownloadHandlers:self._handlers[scheme] = dhreturn dh+ @_warn_spider_argdef download_request(self, request: Request, spider: Spider | None = None) -> Deferred[Response]:- if spider is not None:- warnings.warn(- "Passing a 'spider' argument to DownloadHandlers.download_request() is deprecated.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )scheme = urlparse_cached(request).schemehandler = self._get_handler(scheme)if not handler:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 769b32fc9..9184731cd 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -23,6 +23,7 @@ from scrapy.exceptions import (from scrapy.http import Request, Responsefrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (_defer_sleep_async,_schedule_coro,@@ -219,16 +220,10 @@ class Scraper:self.slot.closing.callback(self.crawler.spider)@inlineCallbacks+ @_warn_spider_argdef enqueue_scrape(self, result: Response | Failure, request: Request, spider: Spider | None = None) -> Generator[Deferred[Any], Any, None]:- if spider is not None:- warnings.warn(- "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-if self.slot is None:raise RuntimeError("Scraper slot not assigned")dfd = self.slot.add_response_request(result, request)@@ -343,6 +338,7 @@ class Scraper:# which needs to be passed to iterate_spider_output()return await ensure_awaitable(iterate_spider_output(output))+ @_warn_spider_argdef handle_spider_error(self,_failure: Failure,@@ -351,13 +347,6 @@ class Scraper:spider: Spider | None = None,) -> None:"""Handle an exception raised by a spider callback or errback."""- if spider is not None:- warnings.warn(- "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-assert self.crawler.spiderexc = _failure.valueif isinstance(exc, CloseSpider):diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex fb65ff06a..a7cd51166 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -22,6 +22,7 @@ from scrapy.http.request import NO_CALLBACK, Requestfrom scrapy.settings import Settingsfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.datatypes import SequenceExclude+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_corofrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iter@@ -165,27 +166,15 @@ class MediaPipeline(ABC):pipe._finish_init(crawler)return pipe+ @_warn_spider_argdef open_spider(self, spider: Spider | None = None) -> None:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to MediaPipeline.open_spider()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )assert self.crawler.spiderself.spiderinfo = self.SpiderInfo(self.crawler.spider)+ @_warn_spider_argdef process_item(self, item: Any, spider: Spider | None = None) -> Deferred[list[FileInfoOrError]]:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to MediaPipeline.process_item()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )info = self.spiderinforequests = arg_to_iter(self.get_media_requests(item, info))dlist = [self._process_request(r, info, item) for r in requests]diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pyindex a047809c3..889fc6df1 100644--- a/scrapy/spidermiddlewares/base.py+++ b/scrapy/spidermiddlewares/base.py@@ -1,10 +1,9 @@from __future__ import annotations-import warningsfrom typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spider-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:from collections.abc import AsyncIterator, Iterable@@ -54,33 +53,21 @@ class BaseSpiderMiddleware:if (o := self._get_processed(o, None)) is not None:yield o+ @_warn_spider_argdef process_spider_output(self, response: Response, result: Iterable[Any], spider: Spider | None = None) -> Iterable[Any]:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )for o in result:if (o := self._get_processed(o, response)) is not None:yield o+ @_warn_spider_argasync def process_spider_output_async(self,response: Response,result: AsyncIterator[Any],spider: Spider | None = None,) -> AsyncIterator[Any]:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )async for o in result:if (o := self._get_processed(o, response)) is not None:yield odiff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex d23bca50c..9c387530c 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -7,11 +7,10 @@ See documentation in docs/topics/spider-middleware.rstfrom __future__ import annotationsimport logging-import warningsfrom typing import TYPE_CHECKING, Any-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.spidermiddlewares.base import BaseSpiderMiddleware+from scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:from collections.abc import AsyncIterator, Iterable@@ -54,32 +53,20 @@ class DepthMiddleware(BaseSpiderMiddleware):o.crawler = crawlerreturn o+ @_warn_spider_argdef process_spider_output(self, response: Response, result: Iterable[Any], spider: Spider | None = None) -> Iterable[Any]:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to DepthMiddleware.process_spider_output()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )self._init_depth(response)yield from super().process_spider_output(response, result)+ @_warn_spider_argasync def process_spider_output_async(self,response: Response,result: AsyncIterator[Any],spider: Spider | None = None,) -> AsyncIterator[Any]:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to DepthMiddleware.process_spider_output_async()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )self._init_depth(response)async for o in super().process_spider_output_async(response, result):yield odiff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex b4c9c10bf..c160d1adb 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -7,10 +7,10 @@ See documentation in docs/topics/spider-middleware.rstfrom __future__ import annotationsimport logging-import warningsfrom typing import TYPE_CHECKING, Any-from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning+from scrapy.exceptions import IgnoreRequest+from scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:from collections.abc import Iterable@@ -50,16 +50,10 @@ class HttpErrorMiddleware:o.crawler = crawlerreturn o+ @_warn_spider_argdef process_spider_input(self, response: Response, spider: Spider | None = None) -> None:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to HttpErrorMiddleware.process_spider_input()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )if 200 <= response.status < 300: # common casereturnmeta = response.meta@@ -79,16 +73,10 @@ class HttpErrorMiddleware:returnraise HttpError(response, "Ignoring non-200 response")+ @_warn_spider_argdef process_spider_exception(self, response: Response, exception: Exception, spider: Spider | None = None) -> Iterable[Any] | None:- if spider is not None: # pragma: no cover- warnings.warn(- "Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"- " is deprecated and the passed value is ignored.",- ScrapyDeprecationWarning,- stacklevel=2,- )if isinstance(exception, HttpError):assert self.crawler.statsself.crawler.stats.inc_value("httperror/response_ignored_count")diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.pyindex c2fa0ae6a..02adef1e4 100644--- a/scrapy/statscollectors.py+++ b/scrapy/statscollectors.py@@ -4,13 +4,11 @@ Scrapy extension for collecting scraping statsfrom __future__ import annotations-import inspectimport loggingimport pprint-import warningsfrom typing import TYPE_CHECKING, Any-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:from scrapy import Spider@@ -44,22 +42,7 @@ class StatsCollector:"open_spider","close_spider",) and callable(original_attr):-- def _deprecated_wrapper(*args, **kwargs):- sig = inspect.signature(original_attr).bind(*args, **kwargs)- sig.apply_defaults()-- if sig.arguments.get("spider"):- warnings.warn(- f"Passing a 'spider' argument to StatsCollector.{name}() is deprecated and"- f" the argument will be removed in a future Scrapy version.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-- return original_attr(*args, **kwargs)-- return _deprecated_wrapper+ return _warn_spider_arg(original_attr)return original_attrdiff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex 0f4d0beda..c4739bcee 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -1,8 +1,9 @@from __future__ import annotations+import inspectimport warningsfrom functools import wraps-from typing import TYPE_CHECKING, Any, TypeVar+from typing import TYPE_CHECKING, Any, TypeVar, overloadfrom twisted.internet.defer import Deferred, maybeDeferredfrom twisted.internet.threads import deferToThread@@ -10,7 +11,7 @@ from twisted.internet.threads import deferToThreadfrom scrapy.exceptions import ScrapyDeprecationWarningif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import AsyncGenerator, Callable, Coroutine# typing.ParamSpec requires Python 3.10from typing_extensions import ParamSpec@@ -30,7 +31,7 @@ def deprecated(def deco(func: Callable[_P, _T]) -> Callable[_P, _T]:@wraps(func)- def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any:+ def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T:message = f"Call to deprecated function {func.__name__}."if use_instead:message += f" Use {use_instead} instead."@@ -65,3 +66,67 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:return deferToThread(func, *a, **kw)return wrapped+++@overload+def _warn_spider_arg(+ func: Callable[_P, Coroutine[Any, Any, _T]],+) -> Callable[_P, Coroutine[Any, Any, _T]]: ...+++@overload+def _warn_spider_arg(+ func: Callable[_P, AsyncGenerator[_T]],+) -> Callable[_P, AsyncGenerator[_T]]: ...+++@overload+def _warn_spider_arg(func: Callable[_P, _T]) -> Callable[_P, _T]: ...+++def _warn_spider_arg(+ func: Callable[_P, _T],+) -> (+ Callable[_P, _T]+ | Callable[_P, Coroutine[Any, Any, _T]]+ | Callable[_P, AsyncGenerator[_T]]+):+ """Decorator to warn if a ``spider`` argument is passed to a function."""++ def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None:+ bound = inspect.signature(func).bind(*args, **kwargs)+ if "spider" in bound.arguments:+ warnings.warn(+ f"Passing a 'spider' argument to {func.__qualname__}() is deprecated and "+ "the argument will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=3,+ )++ if inspect.iscoroutinefunction(func):++ @wraps(func)+ async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:+ check_args(*args, **kwargs)+ return await func(*args, **kwargs)++ return async_inner++ if inspect.isasyncgenfunction(func):++ @wraps(func)+ async def asyncgen_inner(+ *args: _P.args, **kwargs: _P.kwargs+ ) -> AsyncGenerator[_T]:+ check_args(*args, **kwargs)+ async for item in func(*args, **kwargs):+ yield item++ return asyncgen_inner++ @wraps(func)+ def sync_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:+ check_args(*args, **kwargs)+ return func(*args, **kwargs)++ return sync_innerdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 5276636fc..99d5cbb6d 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -180,7 +180,7 @@ class TestFilesPipeline:for p in patchers:p.start()- result = yield self.pipeline.process_item(item, None)+ result = yield self.pipeline.process_item(item)assert result["files"][0]["checksum"] == "abc"assert result["files"][0]["status"] == "uptodate"@@ -211,7 +211,7 @@ class TestFilesPipeline:for p in patchers:p.start()- result = yield self.pipeline.process_item(item, None)+ result = yield self.pipeline.process_item(item)assert result["files"][0]["checksum"] != "abc"assert result["files"][0]["status"] == "downloaded"@@ -242,7 +242,7 @@ class TestFilesPipeline:for p in patchers:p.start()- result = yield self.pipeline.process_item(item, None)+ result = yield self.pipeline.process_item(item)assert result["files"][0]["checksum"] != "abc"assert result["files"][0]["status"] == "cached"diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.pyindex e2ee48e1d..da570904d 100644--- a/tests/test_spidermiddleware_base.py+++ b/tests/test_spidermiddleware_base.py@@ -28,7 +28,7 @@ def test_trivial(crawler: Crawler) -> None:test_req = Request("data:,")spider_output = [test_req, {"foo": "bar"}]for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]+ list(mw.process_spider_output(Response("data:,"), spider_output)),list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]]:assert processed == [test_req, {"foo": "bar"}]@@ -51,7 +51,7 @@ def test_processed_request(crawler: Crawler) -> None:test_req3 = Request("data:3,")spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]+ list(mw.process_spider_output(Response("data:,"), spider_output)),list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]]:assert len(processed) == 3@@ -75,7 +75,7 @@ def test_processed_item(crawler: Crawler) -> None:test_req = Request("data:,")spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]+ list(mw.process_spider_output(Response("data:,"), spider_output)),list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]]:assert processed == [{"foo": 1}, test_req, {"foo": 30}]@@ -112,7 +112,7 @@ def test_processed_both(crawler: Crawler) -> None:test_req3,]for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]+ list(mw.process_spider_output(Response("data:,"), spider_output)),list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]]:assert len(processed) == 4