Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate the spider arg of downloader middleware methods. (#7037)
Andrey Rakhmatullin2ad81a0ef883
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 60b6aab78..6f36acc68 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -70,7 +70,7 @@ defines one or more of these methods:.. note:: Any of the downloader middleware methods may also return a deferred.- .. method:: process_request(request, spider)+ .. method:: process_request(request)This method is called for each request that goes through the downloadmiddleware.@@ -102,10 +102,7 @@ defines one or more of these methods::param request: the request being processed:type request: :class:`~scrapy.Request` object- :param spider: the spider for which this request is intended- :type spider: :class:`~scrapy.Spider` object-- .. method:: process_response(request, response, spider)+ .. method:: process_response(request, response):meth:`process_response` should either: return a :class:`~scrapy.http.Response`object, return a :class:`~scrapy.Request` object or@@ -129,10 +126,7 @@ defines one or more of these methods::param response: the response being processed:type response: :class:`~scrapy.http.Response` object- :param spider: the spider for which this response is intended- :type spider: :class:`~scrapy.Spider` object-- .. method:: process_exception(request, exception, spider)+ .. method:: process_exception(request, exception)Scrapy calls :meth:`process_exception` when a download handleror a :meth:`process_request` (from a downloader middleware) raises an@@ -160,9 +154,6 @@ defines one or more of these methods::param exception: the raised exception:type exception: an ``Exception`` object- :param spider: the spider for which this request is intended- :type spider: :class:`~scrapy.Spider` object-.. _topics-downloader-middleware-ref:Built-in downloader middleware referencediff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex 47175fb13..a822ba811 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -35,10 +35,13 @@ class DownloaderMiddlewareManager(MiddlewareManager):def _add_middleware(self, mw: Any) -> None:if hasattr(mw, "process_request"):self.methods["process_request"].append(mw.process_request)+ self._check_mw_method_spider_arg(mw.process_request)if hasattr(mw, "process_response"):self.methods["process_response"].appendleft(mw.process_response)+ self._check_mw_method_spider_arg(mw.process_response)if hasattr(mw, "process_exception"):self.methods["process_exception"].appendleft(mw.process_exception)+ self._check_mw_method_spider_arg(mw.process_exception)@inlineCallbacksdef download(@@ -64,9 +67,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):) -> Generator[Deferred[Any], Any, Response | Request]:for method in self.methods["process_request"]:method = cast("Callable", method)- response = yield deferred_from_coro(- method(request=request, spider=self._spider)- )+ if method in self._mw_methods_requiring_spider:+ response = yield deferred_from_coro(+ method(request=request, spider=self._spider)+ )+ else:+ response = yield deferred_from_coro(method(request=request))if response is not None and not isinstance(response, (Response, Request)):@@ -91,9 +97,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):for method in self.methods["process_response"]:method = cast("Callable", method)- response = yield deferred_from_coro(- method(request=request, response=response, spider=self._spider)- )+ if method in self._mw_methods_requiring_spider:+ response = yield deferred_from_coro(+ method(request=request, response=response, spider=self._spider)+ )+ else:+ response = yield deferred_from_coro(+ method(request=request, response=response)+ )if not isinstance(response, (Response, Request)):raise _InvalidOutput(f"Middleware {method.__qualname__} must return Response or Request, "@@ -109,9 +120,16 @@ class DownloaderMiddlewareManager(MiddlewareManager):) -> Generator[Deferred[Any], Any, Response | Request]:for method in self.methods["process_exception"]:method = cast("Callable", method)- response = yield deferred_from_coro(- method(request=request, exception=exception, spider=self._spider)- )+ if method in self._mw_methods_requiring_spider:+ response = yield deferred_from_coro(+ method(+ request=request, exception=exception, spider=self._spider+ )+ )+ else:+ response = yield deferred_from_coro(+ method(request=request, exception=exception)+ )if response is not None and not isinstance(response, (Response, Request)):diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 9156b8c3a..d945546d5 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -9,6 +9,7 @@ from tldextract import TLDExtractfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Responsefrom scrapy.http.cookies import CookieJar+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_unicode@@ -39,6 +40,8 @@ def _is_public_domain(domain: str) -> bool:class CookiesMiddleware:"""This middleware enables working with sites that need cookies"""+ crawler: Crawler+def __init__(self, debug: bool = False):self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar)self.debug: bool = debug@@ -47,7 +50,9 @@ class CookiesMiddleware:def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("COOKIES_ENABLED"):raise NotConfigured- return cls(crawler.settings.getbool("COOKIES_DEBUG"))+ o = cls(crawler.settings.getbool("COOKIES_DEBUG"))+ o.crawler = crawler+ return odef _process_cookies(self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request@@ -67,8 +72,9 @@ class CookiesMiddleware:jar.set_cookie_if_ok(cookie, request)+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:if request.meta.get("dont_merge_cookies", False):return None@@ -81,11 +87,12 @@ class CookiesMiddleware:# set Cookie headerrequest.headers.pop("Cookie", None)jar.add_cookie_header(request)- self._debug_cookie(request, spider)+ self._debug_cookie(request)return None+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if request.meta.get("dont_merge_cookies", False):return response@@ -96,11 +103,11 @@ class CookiesMiddleware:cookies = jar.make_cookies(response, request)self._process_cookies(cookies, jar=jar, request=request)- self._debug_set_cookie(response, spider)+ self._debug_set_cookie(response)return response- def _debug_cookie(self, request: Request, spider: Spider) -> None:+ def _debug_cookie(self, request: Request) -> None:if self.debug:cl = [to_unicode(c, errors="replace")@@ -109,9 +116,9 @@ class CookiesMiddleware:if cl:cookies = "\n".join(f"Cookie: {c}\n" for c in cl)msg = f"Sending cookies to: {request}\n{cookies}"- logger.debug(msg, extra={"spider": spider})+ logger.debug(msg, extra={"spider": self.crawler.spider})- def _debug_set_cookie(self, response: Response, spider: Spider) -> None:+ def _debug_set_cookie(self, response: Response) -> None:if self.debug:cl = [to_unicode(c, errors="replace")@@ -120,7 +127,7 @@ class CookiesMiddleware:if cl:cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)msg = f"Received cookies from: {response}\n{cookies}"- logger.debug(msg, extra={"spider": spider})+ logger.debug(msg, extra={"spider": self.crawler.spider})def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None:"""diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.pyindex d58b4490b..7c940a078 100644--- a/scrapy/downloadermiddlewares/defaultheaders.py+++ b/scrapy/downloadermiddlewares/defaultheaders.py@@ -8,6 +8,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.python import without_none_valuesif TYPE_CHECKING:@@ -30,8 +31,9 @@ class DefaultHeadersMiddleware:headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])return cls(headers.items())+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:for k, v in self._headers:request.headers.setdefault(k, v)diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.pyindex 28456c697..b57d5c2a9 100644--- a/scrapy/downloadermiddlewares/downloadtimeout.py+++ b/scrapy/downloadermiddlewares/downloadtimeout.py@@ -9,6 +9,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom scrapy import Request, Spider, signals+from scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:# typing.Self requires Python 3.11@@ -31,8 +32,9 @@ class DownloadTimeoutMiddleware:def spider_opened(self, spider: Spider) -> None:self._timeout = getattr(spider, "download_timeout", self._timeout)+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:if self._timeout:request.meta.setdefault("download_timeout", self._timeout)diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.pyindex 80107261b..c28c93d4e 100644--- a/scrapy/downloadermiddlewares/httpauth.py+++ b/scrapy/downloadermiddlewares/httpauth.py@@ -11,6 +11,7 @@ from typing import TYPE_CHECKINGfrom w3lib.http import basic_auth_headerfrom scrapy import Request, Spider, signals+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.url import url_is_from_any_domainif TYPE_CHECKING:@@ -38,8 +39,9 @@ class HttpAuthMiddleware:self.auth = basic_auth_header(usr, pwd)self.domain = spider.http_auth_domain # type: ignore[attr-defined]+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:auth = getattr(self, "auth", None)if (diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex d3bfe6eba..453e8a6a3 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -17,6 +17,7 @@ from twisted.web.client import ResponseFailedfrom scrapy import signalsfrom scrapy.exceptions import IgnoreRequest, NotConfigured+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.misc import load_objectif TYPE_CHECKING:@@ -45,6 +46,8 @@ class HttpCacheMiddleware:OSError,)+ crawler: Crawler+def __init__(self, settings: Settings, stats: StatsCollector) -> None:if not settings.getbool("HTTPCACHE_ENABLED"):raise NotConfigured@@ -59,6 +62,7 @@ class HttpCacheMiddleware:o = cls(crawler.settings, crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)+ o.crawler = crawlerreturn odef spider_opened(self, spider: Spider) -> None:@@ -67,8 +71,9 @@ class HttpCacheMiddleware:def spider_closed(self, spider: Spider) -> None:self.storage.close_spider(spider)+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:if request.meta.get("dont_cache", False):return None@@ -80,7 +85,7 @@ class HttpCacheMiddleware:# Look for cached response and check if expiredcachedresponse: Response | None = self.storage.retrieve_response(- spider, request+ self.crawler.spider, request)if cachedresponse is None:self.stats.inc_value("httpcache/miss")@@ -101,8 +106,9 @@ class HttpCacheMiddleware:return None+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if request.meta.get("dont_cache", False):return response@@ -121,7 +127,7 @@ class HttpCacheMiddleware:cachedresponse: Response | None = request.meta.pop("cached_response", None)if cachedresponse is None:self.stats.inc_value("httpcache/firsthand")- self._cache_response(spider, response, request, cachedresponse)+ self._cache_response(response, request)return responseif self.policy.is_cached_response_valid(cachedresponse, response, request):@@ -129,11 +135,12 @@ class HttpCacheMiddleware:return cachedresponseself.stats.inc_value("httpcache/invalidate")- self._cache_response(spider, response, request, cachedresponse)+ self._cache_response(response, request)return response+ @_warn_spider_argdef process_exception(- self, request: Request, exception: Exception, spider: Spider+ self, request: Request, exception: Exception, spider: Spider | None = None) -> Request | Response | None:cachedresponse: Response | None = request.meta.pop("cached_response", None)if cachedresponse is not None and isinstance(@@ -143,15 +150,9 @@ class HttpCacheMiddleware:return cachedresponsereturn None- def _cache_response(- self,- spider: Spider,- response: Response,- request: Request,- cachedresponse: Response | None,- ) -> None:+ def _cache_response(self, response: Response, request: Request) -> None:if self.policy.should_cache_response(response, request):self.stats.inc_value("httpcache/store")- self.storage.store_response(spider, request, response)+ self.storage.store_response(self.crawler.spider, request, response)else:self.stats.inc_value("httpcache/uncacheable")diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex e3326146e..f729649a0 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -15,6 +15,7 @@ from scrapy.utils._compression import (_unbrotli,_unzstd,)+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.gz import gunzipif TYPE_CHECKING:@@ -93,14 +94,16 @@ class HttpCompressionMiddleware:)self._warn_size = spider.download_warnsize+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))return None+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if request.method == "HEAD":return responsediff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex cb7fa8c90..d3d46a947 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -10,6 +10,7 @@ from urllib.request import ( # type: ignore[attr-defined])from scrapy.exceptions import NotConfigured+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes@@ -55,8 +56,9 @@ class HttpProxyMiddleware:return creds, proxy_url+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:creds, proxy_url, scheme = None, None, Noneif "proxy" in request.meta:diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex 682f2cb06..33d7ba609 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -7,6 +7,7 @@ from typing import TYPE_CHECKINGfrom scrapy import Request, Spider, signalsfrom scrapy.exceptions import IgnoreRequest+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:@@ -21,29 +22,34 @@ logger = logging.getLogger(__name__)class OffsiteMiddleware:+ crawler: Crawler++ def __init__(self, stats: StatsCollector):+ self.stats = stats+ self.domains_seen: set[str] = set()+@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:assert crawler.statso = cls(crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)+ o.crawler = crawlerreturn o- def __init__(self, stats: StatsCollector):- self.stats = stats- self.domains_seen: set[str] = set()-def spider_opened(self, spider: Spider) -> None:self.host_regex: re.Pattern[str] = self.get_host_regex(spider)def request_scheduled(self, request: Request, spider: Spider) -> None:- self.process_request(request, spider)+ self.process_request(request)- def process_request(self, request: Request, spider: Spider) -> None:+ @_warn_spider_arg+ def process_request(self, request: Request, spider: Spider | None = None) -> None:+ assert self.crawler.spiderif (request.dont_filteror request.meta.get("allow_offsite")- or self.should_follow(request, spider)+ or self.should_follow(request, self.crawler.spider)):returndomain = urlparse_cached(request).hostname@@ -52,7 +58,7 @@ class OffsiteMiddleware:logger.debug("Filtered offsite request to %(domain)r: %(request)s",{"domain": domain, "request": request},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)self.stats.inc_value("offsite/domains")self.stats.inc_value("offsite/filtered")diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 530cccb53..61f62ecfc 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -8,6 +8,7 @@ from w3lib.url import safe_url_stringfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import HtmlResponse, Response+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.response import get_meta_refresh@@ -79,6 +80,7 @@ def _build_redirect_request(class BaseRedirectMiddleware:+ crawler: Crawlerenabled_setting: str = "REDIRECT_ENABLED"def __init__(self, settings: BaseSettings):@@ -90,11 +92,11 @@ class BaseRedirectMiddleware:@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- return cls(crawler.settings)+ o = cls(crawler.settings)+ o.crawler = crawler+ return o- def _redirect(- self, redirected: Request, request: Request, spider: Spider, reason: Any- ) -> Request:+ def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request:ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)redirects = request.meta.get("redirect_times", 0) + 1@@ -114,13 +116,13 @@ class BaseRedirectMiddleware:logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",{"reason": reason, "redirected": redirected, "request": request},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)return redirectedlogger.debug("Discarding %(request)s: max redirections reached",{"request": request},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)raise IgnoreRequest("max redirections reached")@@ -144,12 +146,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):and meta-refresh html tag."""+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if (request.meta.get("dont_redirect", False)- or response.status in getattr(spider, "handle_httpstatus_list", [])+ or response.status+ in getattr(self.crawler.spider, "handle_httpstatus_list", [])or response.status in request.meta.get("handle_httpstatus_list", [])or request.meta.get("handle_httpstatus_all", False)):@@ -171,10 +175,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):return responseif response.status in (301, 307, 308) or request.method == "HEAD":- return self._redirect(redirected, request, spider, response.status)+ return self._redirect(redirected, request, response.status)redirected = self._redirect_request_using_get(request, redirected_url)- return self._redirect(redirected, request, spider, response.status)+ return self._redirect(redirected, request, response.status)class MetaRefreshMiddleware(BaseRedirectMiddleware):@@ -185,8 +189,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if (request.meta.get("dont_redirect", False)@@ -203,5 +208,5 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):if urlparse_cached(redirected).scheme not in {"http", "https"}:return responseif cast("float", interval) < self._maxdelay:- return self._redirect(redirected, request, spider, "meta refresh")+ return self._redirect(redirected, request, "meta refresh")return responsediff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 723fe5e93..b6dba5773 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -16,6 +16,7 @@ from logging import Logger, getLoggerfrom typing import TYPE_CHECKINGfrom scrapy.exceptions import NotConfigured+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import global_object_namefrom scrapy.utils.response import response_status_message@@ -123,6 +124,8 @@ def get_retry_request(class RetryMiddleware:+ crawler: Crawler+def __init__(self, settings: BaseSettings):if not settings.getbool("RETRY_ENABLED"):raise NotConfigured@@ -136,39 +139,41 @@ class RetryMiddleware:@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:- return cls(crawler.settings)+ o = cls(crawler.settings)+ o.crawler = crawler+ return o+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:if request.meta.get("dont_retry", False):return responseif response.status in self.retry_http_codes:reason = response_status_message(response.status)- return self._retry(request, reason, spider) or response+ return self._retry(request, reason) or responsereturn response+ @_warn_spider_argdef process_exception(- self, request: Request, exception: Exception, spider: Spider+ self, request: Request, exception: Exception, spider: Spider | None = None) -> Request | Response | None:if isinstance(exception, self.exceptions_to_retry) and not request.meta.get("dont_retry", False):- return self._retry(request, exception, spider)+ return self._retry(request, exception)return Nonedef _retry(- self,- request: Request,- reason: str | Exception | type[Exception],- spider: Spider,+ self, request: Request, reason: str | Exception | type[Exception]) -> Request | None:max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)+ assert self.crawler.spiderreturn get_retry_request(request,reason=reason,- spider=spider,+ spider=self.crawler.spider,max_retry_times=max_retry_times,priority_adjust=priority_adjust,)diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 83af0f7bf..7d0c17884 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -14,6 +14,7 @@ from twisted.internet.defer import Deferredfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_object@@ -51,17 +52,18 @@ class RobotsTxtMiddleware:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler)- async def process_request(self, request: Request, spider: Spider) -> None:+ @_warn_spider_arg+ async def process_request(+ self, request: Request, spider: Spider | None = None+ ) -> None:if request.meta.get("dont_obey_robotstxt"):returnif request.url.startswith("data:") or request.url.startswith("file:"):return- rp = await self.robot_parser(request, spider)- self.process_request_2(rp, request, spider)+ rp = await self.robot_parser(request)+ self.process_request_2(rp, request)- def process_request_2(- self, rp: RobotParser | None, request: Request, spider: Spider- ) -> None:+ def process_request_2(self, rp: RobotParser | None, request: Request) -> None:if rp is None:return@@ -73,15 +75,13 @@ class RobotsTxtMiddleware:logger.debug("Forbidden by robots.txt: %(request)s",{"request": request},- extra={"spider": spider},+ extra={"spider": self.crawler.spider},)assert self.crawler.statsself.crawler.stats.inc_value("robotstxt/forbidden")raise IgnoreRequest("Forbidden by robots.txt")- async def robot_parser(- self, request: Request, spider: Spider- ) -> RobotParser | None:+ async def robot_parser(self, request: Request) -> RobotParser | None:url = urlparse_cached(request)netloc = url.netloc@@ -100,7 +100,13 @@ class RobotsTxtMiddleware:resp = await self.crawler.engine.download_async(robotsreq)self._parse_robots(resp, netloc)except Exception as e:- self._logerror(e, robotsreq, spider)+ if not isinstance(e, IgnoreRequest):+ logger.error(+ "Error downloading %(request)s: %(f_exception)s",+ {"request": request, "f_exception": e},+ exc_info=True,+ extra={"spider": self.crawler.spider},+ )self._robots_error(e, netloc)self.crawler.stats.inc_value("robotstxt/request_count")@@ -109,15 +115,6 @@ class RobotsTxtMiddleware:return await maybe_deferred_to_future(parser)return parser- def _logerror(self, exc: Exception, request: Request, spider: Spider) -> None:- if not isinstance(exc, IgnoreRequest):- logger.error(- "Error downloading %(request)s: %(f_exception)s",- {"request": request, "f_exception": exc},- exc_info=True, # noqa: LOG014- extra={"spider": spider},- )-def _parse_robots(self, response: Response, netloc: str) -> None:assert self.crawler.statsself.crawler.stats.inc_value("robotstxt/response_count")diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex 38e9ad891..bafa931de 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKINGfrom twisted.web import httpfrom scrapy.exceptions import NotConfigured+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.python import global_object_name, to_bytesfrom scrapy.utils.request import request_httprepr@@ -45,8 +46,9 @@ class DownloaderStats:assert crawler.statsreturn cls(crawler.stats)+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:self.stats.inc_value("downloader/request_count")self.stats.inc_value(f"downloader/request_method_count/{request.method}")@@ -54,8 +56,9 @@ class DownloaderStats:self.stats.inc_value("downloader/request_bytes", reqlen)return None+ @_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider+ self, request: Request, response: Response, spider: Spider | None = None) -> Request | Response:self.stats.inc_value("downloader/response_count")self.stats.inc_value(f"downloader/response_status_count/{response.status}")@@ -69,8 +72,9 @@ class DownloaderStats:self.stats.inc_value("downloader/response_bytes", reslen)return response+ @_warn_spider_argdef process_exception(- self, request: Request, exception: Exception, spider: Spider+ self, request: Request, exception: Exception, spider: Spider | None = None) -> Request | Response | None:ex_class = global_object_name(exception.__class__)self.stats.inc_value("downloader/exception_count")diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.pyindex ba379f862..c43a0195c 100644--- a/scrapy/downloadermiddlewares/useragent.py+++ b/scrapy/downloadermiddlewares/useragent.py@@ -5,6 +5,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom scrapy import Request, Spider, signals+from scrapy.utils.decorators import _warn_spider_argif TYPE_CHECKING:# typing.Self requires Python 3.11@@ -29,8 +30,9 @@ class UserAgentMiddleware:def spider_opened(self, spider: Spider) -> None:self.user_agent = getattr(spider, "user_agent", self.user_agent)+ @_warn_spider_argdef process_request(- self, request: Request, spider: Spider+ self, request: Request, spider: Spider | None = None) -> Request | Response | None:if self.user_agent:request.headers.setdefault(b"User-Agent", self.user_agent)diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 2684cc1de..f50e7bec7 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -128,7 +128,7 @@ class TestResponseFromProcessRequest(TestManagerBase):download_func = mock.MagicMock()class ResponseMiddleware:- def process_request(self, request, spider):+ def process_request(self, request):return respasync with self.get_mwman() as mwman:@@ -151,11 +151,11 @@ class TestResponseFromProcessException(TestManagerBase):raise ValueError("test")class ResponseMiddleware:- def process_response(self, request, response, spider):+ def process_response(self, request, response):calls.append("process_response")return resp- def process_exception(self, request, exception, spider):+ def process_exception(self, request, exception):calls.append("process_exception")return resp@@ -176,7 +176,7 @@ class TestInvalidOutput(TestManagerBase):req = Request("http://example.com/index.html")class InvalidProcessRequestMiddleware:- def process_request(self, request, spider):+ def process_request(self, request):return 1async with self.get_mwman() as mwman:@@ -190,7 +190,7 @@ class TestInvalidOutput(TestManagerBase):req = Request("http://example.com/index.html")class InvalidProcessResponseMiddleware:- def process_response(self, request, response, spider):+ def process_response(self, request, response):return 1async with self.get_mwman() as mwman:@@ -204,10 +204,10 @@ class TestInvalidOutput(TestManagerBase):req = Request("http://example.com/index.html")class InvalidProcessExceptionMiddleware:- def process_request(self, request, spider):+ def process_request(self, request):raise RuntimeError- def process_exception(self, request, exception, spider):+ def process_exception(self, request, exception):return 1async with self.get_mwman() as mwman:@@ -229,7 +229,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):def cb(self, result):return result- def process_request(self, request, spider):+ def process_request(self, request):d = Deferred()d.addCallback(self.cb)d.callback(resp)@@ -252,7 +252,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):download_func = mock.MagicMock()class CoroMiddleware:- async def process_request(self, request, spider):+ async def process_request(self, request):await succeed(42)return resp@@ -270,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):download_func = mock.MagicMock()class CoroMiddleware:- async def process_request(self, request, spider):+ async def process_request(self, request):await asyncio.sleep(0.1)return await get_from_asyncio_queue(resp)@@ -315,3 +315,41 @@ class TestDownloadDeprecated(TestManagerBase):mwman.download(download_func, req, mwman.crawler.spider))assert isinstance(ret, Response)+++class TestDeprecatedSpiderArg(TestManagerBase):+ @deferred_f_from_coro_f+ async def test_deprecated_spider_arg(self):+ req = Request("http://example.com/index.html")+ resp = Response("http://example.com/index.html")+ download_func = mock.MagicMock()++ class DeprecatedSpiderArgMiddleware:+ def process_request(self, request, spider):+ 1 / 0++ def process_response(self, request, response, spider):+ return response++ def process_exception(self, request, exception, spider):+ return resp++ async with self.get_mwman() as mwman:+ with (+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"process_request\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"process_response\(\) requires a spider argument",+ ),+ pytest.warns(+ ScrapyDeprecationWarning,+ match=r"process_exception\(\) requires a spider argument",+ ),+ ):+ mwman._add_middleware(DeprecatedSpiderArgMiddleware())+ result = await maybe_deferred_to_future(mwman.download(download_func, req))+ assert result is resp+ assert not download_func.calleddiff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 8bf3a1f09..49215329e 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -8,9 +8,8 @@ from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddlewarefrom scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Response-from scrapy.settings import Settings-from scrapy.spiders import Spiderfrom scrapy.utils.python import to_bytes+from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerUNSET = object()@@ -60,9 +59,10 @@ class TestCookiesMiddleware:assert split_cookies(first) == split_cookies(second), msgdef setup_method(self):- self.spider = Spider("foo")- self.mw = CookiesMiddleware()- self.redirect_middleware = RedirectMiddleware(settings=Settings())+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ self.mw = CookiesMiddleware.from_crawler(crawler)+ self.redirect_middleware = RedirectMiddleware.from_crawler(crawler)def teardown_method(self):del self.mw@@ -70,15 +70,15 @@ class TestCookiesMiddleware:def test_basic(self):req = Request("http://scrapytest.org/")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert "Cookie" not in req.headersheaders = {"Set-Cookie": "C1=value1; path=/"}res = Response("http://scrapytest.org/", headers=headers)- assert self.mw.process_response(req, res, self.spider) is res+ assert self.mw.process_response(req, res) is resreq2 = Request("http://scrapytest.org/sub1/")- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneassert req2.headers.get("Cookie") == b"C1=value1"def test_setting_false_cookies_enabled(self):@@ -112,9 +112,9 @@ class TestCookiesMiddleware:res = Response("http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"})- mw.process_response(req, res, crawler.spider)+ mw.process_response(req, res)req2 = Request("http://scrapytest.org/sub1/")- mw.process_request(req2, crawler.spider)+ mw.process_request(req2)log.check((@@ -143,23 +143,23 @@ class TestCookiesMiddleware:res = Response("http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"})- mw.process_response(req, res, crawler.spider)+ mw.process_response(req, res)req2 = Request("http://scrapytest.org/sub1/")- mw.process_request(req2, crawler.spider)+ mw.process_request(req2)log.check()def test_do_not_break_on_non_utf8_header(self):req = Request("http://scrapytest.org/")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert "Cookie" not in req.headersheaders = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"}res = Response("http://scrapytest.org/", headers=headers)- assert self.mw.process_response(req, res, self.spider) is res+ assert self.mw.process_response(req, res) is resreq2 = Request("http://scrapytest.org/sub1/")- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneassert "Cookie" in req2.headersdef test_dont_merge_cookies(self):@@ -167,11 +167,11 @@ class TestCookiesMiddleware:headers = {"Set-Cookie": "C1=value1; path=/"}req = Request("http://scrapytest.org/")res = Response("http://scrapytest.org/", headers=headers)- assert self.mw.process_response(req, res, self.spider) is res+ assert self.mw.process_response(req, res) is res# test Cookie header is not seted to requestreq = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1})- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert "Cookie" not in req.headers# check that returned cookies are not merged back to jar@@ -179,16 +179,16 @@ class TestCookiesMiddleware:"http://scrapytest.org/dontmerge",headers={"Set-Cookie": "dont=mergeme; path=/"},)- assert self.mw.process_response(req, res, self.spider) is res+ assert self.mw.process_response(req, res) is res# check that cookies are merged backreq = Request("http://scrapytest.org/mergeme")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers.get("Cookie") == b"C1=value1"# check that cookies are merged when dont_merge_cookies is passed as 0req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0})- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers.get("Cookie") == b"C1=value1"def test_complex_cookies(self):@@ -216,11 +216,11 @@ class TestCookiesMiddleware:]req = Request("http://scrapytest.org/", cookies=cookies)- self.mw.process_request(req, self.spider)+ self.mw.process_request(req)# embed C1 and C3 for scrapytest.org/fooreq = Request("http://scrapytest.org/foo")- self.mw.process_request(req, self.spider)+ self.mw.process_request(req)assert req.headers.get("Cookie") in (b"C1=value1; C3=value3",b"C3=value3; C1=value1",@@ -228,25 +228,25 @@ class TestCookiesMiddleware:# embed C2 for scrapytest.org/barreq = Request("http://scrapytest.org/bar")- self.mw.process_request(req, self.spider)+ self.mw.process_request(req)assert req.headers.get("Cookie") == b"C2=value2"# embed nothing for scrapytest.org/bazreq = Request("http://scrapytest.org/baz")- self.mw.process_request(req, self.spider)+ self.mw.process_request(req)assert "Cookie" not in req.headersdef test_merge_request_cookies(self):req = Request("http://scrapytest.org/", cookies={"galleta": "salada"})- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers.get("Cookie") == b"galleta=salada"headers = {"Set-Cookie": "C1=value1; path=/"}res = Response("http://scrapytest.org/", headers=headers)- assert self.mw.process_response(req, res, self.spider) is res+ assert self.mw.process_response(req, res) is resreq2 = Request("http://scrapytest.org/sub1/")- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers.get("Cookie"), b"C1=value1; galleta=salada"@@ -258,15 +258,15 @@ class TestCookiesMiddleware:cookies={"galleta": "salada"},meta={"cookiejar": "store1"},)- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers.get("Cookie") == b"galleta=salada"headers = {"Set-Cookie": "C1=value1; path=/"}res = Response("http://scrapytest.org/", headers=headers, request=req)- assert self.mw.process_response(req, res, self.spider) is res+ assert self.mw.process_response(req, res) is resreq2 = Request("http://scrapytest.org/", meta=res.meta)- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers.get("Cookie"), b"C1=value1; galleta=salada")@@ -276,45 +276,45 @@ class TestCookiesMiddleware:cookies={"galleta": "dulce"},meta={"cookiejar": "store2"},)- assert self.mw.process_request(req3, self.spider) is None+ assert self.mw.process_request(req3) is Noneassert req3.headers.get("Cookie") == b"galleta=dulce"headers = {"Set-Cookie": "C2=value2; path=/"}res2 = Response("http://scrapytest.org/", headers=headers, request=req3)- assert self.mw.process_response(req3, res2, self.spider) is res2+ assert self.mw.process_response(req3, res2) is res2req4 = Request("http://scrapytest.org/", meta=res2.meta)- assert self.mw.process_request(req4, self.spider) is None+ assert self.mw.process_request(req4) is Noneself.assertCookieValEqual(req4.headers.get("Cookie"), b"C2=value2; galleta=dulce")# cookies from hosts with portreq5_1 = Request("http://scrapytest.org:1104/")- assert self.mw.process_request(req5_1, self.spider) is None+ assert self.mw.process_request(req5_1) is Noneheaders = {"Set-Cookie": "C1=value1; path=/"}res5_1 = Response("http://scrapytest.org:1104/", headers=headers, request=req5_1)- assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1+ assert self.mw.process_response(req5_1, res5_1) is res5_1req5_2 = Request("http://scrapytest.org:1104/some-redirected-path")- assert self.mw.process_request(req5_2, self.spider) is None+ assert self.mw.process_request(req5_2) is Noneassert req5_2.headers.get("Cookie") == b"C1=value1"req5_3 = Request("http://scrapytest.org/some-redirected-path")- assert self.mw.process_request(req5_3, self.spider) is None+ assert self.mw.process_request(req5_3) is Noneassert req5_3.headers.get("Cookie") == b"C1=value1"# skip cookie retrieval for not http requestreq6 = Request("file:///scrapy/sometempfile")- assert self.mw.process_request(req6, self.spider) is None+ assert self.mw.process_request(req6) is Noneassert req6.headers.get("Cookie") is Nonedef test_local_domain(self):request = Request("http://example-host/", cookies={"currencyCookie": "USD"})- assert self.mw.process_request(request, self.spider) is None+ assert self.mw.process_request(request) is Noneassert "Cookie" in request.headersassert request.headers["Cookie"] == b"currencyCookie=USD"@@ -324,15 +324,15 @@ class TestCookiesMiddleware:mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())# overwrite with values from 'cookies' request argumentreq1 = Request("http://example.org", cookies={"default": "something"})- assert mw_default_headers.process_request(req1, self.spider) is None- assert self.mw.process_request(req1, self.spider) is None+ assert mw_default_headers.process_request(req1) is None+ assert self.mw.process_request(req1) is Noneself.assertCookieValEqual(req1.headers["Cookie"], b"default=something; asdf=qwerty")# keep bothreq2 = Request("http://example.com", cookies={"a": "b"})- assert mw_default_headers.process_request(req2, self.spider) is None- assert self.mw.process_request(req2, self.spider) is None+ assert mw_default_headers.process_request(req2) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty")@@ -341,13 +341,13 @@ class TestCookiesMiddleware:def test_keep_cookie_header(self):# keep only cookies from 'Cookie' request headerreq1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"})- assert self.mw.process_request(req1, self.spider) is None+ assert self.mw.process_request(req1) is Noneself.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d")# keep cookies from both 'Cookie' request header and 'cookies' keywordreq2 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"})- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f")# overwrite values from 'Cookie' request header with 'cookies' keywordreq3 = Request(@@ -355,40 +355,40 @@ class TestCookiesMiddleware:headers={"Cookie": "a=b; c=d"},cookies={"a": "new", "e": "f"},)- assert self.mw.process_request(req3, self.spider) is None+ assert self.mw.process_request(req3) is Noneself.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f")def test_request_cookies_encoding(self):# 1) UTF8-encoded bytesreq1 = Request("http://example.org", cookies={"a": "á".encode()})- assert self.mw.process_request(req1, self.spider) is None+ assert self.mw.process_request(req1) is Noneself.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")# 2) Non UTF8-encoded bytesreq2 = Request("http://example.org", cookies={"a": "á".encode("latin1")})- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")# 3) Stringreq3 = Request("http://example.org", cookies={"a": "á"})- assert self.mw.process_request(req3, self.spider) is None+ assert self.mw.process_request(req3) is Noneself.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")@pytest.mark.xfail(reason="Cookie header is not currently being processed")def test_request_headers_cookie_encoding(self):# 1) UTF8-encoded bytesreq1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})- assert self.mw.process_request(req1, self.spider) is None+ assert self.mw.process_request(req1) is Noneself.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")# 2) Non UTF8-encoded bytesreq2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")})- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1")# 3) Stringreq3 = Request("http://example.org", headers={"Cookie": "a=á"})- assert self.mw.process_request(req3, self.spider) is None+ assert self.mw.process_request(req3) is Noneself.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")def test_invalid_cookies(self):@@ -402,13 +402,13 @@ class TestCookiesMiddleware:) as lc:cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}]req1 = Request("http://example.org/1", cookies=cookies1)- assert self.mw.process_request(req1, self.spider) is None+ assert self.mw.process_request(req1) is Nonecookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}]req2 = Request("http://example.org/2", cookies=cookies2)- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Nonecookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}]req3 = Request("http://example.org/3", cookies=cookies3)- assert self.mw.process_request(req3, self.spider) is None+ assert self.mw.process_request(req3) is Nonelc.check(("scrapy.downloadermiddlewares.cookies",@@ -436,22 +436,22 @@ class TestCookiesMiddleware:def test_primitive_type_cookies(self):# Booleanreq1 = Request("http://example.org", cookies={"a": True})- assert self.mw.process_request(req1, self.spider) is None+ assert self.mw.process_request(req1) is Noneself.assertCookieValEqual(req1.headers["Cookie"], b"a=True")# Floatreq2 = Request("http://example.org", cookies={"a": 9.5})- assert self.mw.process_request(req2, self.spider) is None+ assert self.mw.process_request(req2) is Noneself.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5")# Integerreq3 = Request("http://example.org", cookies={"a": 10})- assert self.mw.process_request(req3, self.spider) is None+ assert self.mw.process_request(req3) is Noneself.assertCookieValEqual(req3.headers["Cookie"], b"a=10")# Stringreq4 = Request("http://example.org", cookies={"a": "b"})- assert self.mw.process_request(req4, self.spider) is None+ assert self.mw.process_request(req4) is Noneself.assertCookieValEqual(req4.headers["Cookie"], b"a=b")def _test_cookie_redirect(@@ -471,7 +471,7 @@ class TestCookiesMiddleware:target.setdefault("status", 301)request1 = Request(cookies=input_cookies, **source)- self.mw.process_request(request1, self.spider)+ self.mw.process_request(request1)cookies = request1.headers.get("Cookie")assert cookies == (b"a=b" if cookies1 else None)@@ -481,16 +481,12 @@ class TestCookiesMiddleware:},**target,)- assert self.mw.process_response(request1, response, self.spider) == response+ assert self.mw.process_response(request1, response) == response- request2 = self.redirect_middleware.process_response(- request1,- response,- self.spider,- )+ request2 = self.redirect_middleware.process_response(request1, response)assert isinstance(request2, Request)- self.mw.process_request(request2, self.spider)+ self.mw.process_request(request2)cookies = request2.headers.get("Cookie")assert cookies == (b"a=b" if cookies2 else None)@@ -564,11 +560,7 @@ class TestCookiesMiddleware:**target,)- request2 = self.redirect_middleware.process_response(- request1,- response,- self.spider,- )+ request2 = self.redirect_middleware.process_response(request1, response)assert isinstance(request2, Request)cookies = request2.headers.get("Cookie")@@ -620,12 +612,12 @@ class TestCookiesMiddleware:]request1 = Request(url1, cookies=input_cookies)- self.mw.process_request(request1, self.spider)+ self.mw.process_request(request1)cookies = request1.headers.get("Cookie")assert cookies == (b"a=b" if cookies1 else None)request2 = Request(url2)- self.mw.process_request(request2, self.spider)+ self.mw.process_request(request2)cookies = request2.headers.get("Cookie")assert cookies == (b"a=b" if cookies2 else None)@@ -674,7 +666,7 @@ class TestCookiesMiddleware:cookies,):request1 = Request(url1)- self.mw.process_request(request1, self.spider)+ self.mw.process_request(request1)input_cookies = [{@@ -688,10 +680,10 @@ class TestCookiesMiddleware:"Set-Cookie": _cookies_to_set_cookie_list(input_cookies),}response = Response(url1, status=200, headers=headers)- assert self.mw.process_response(request1, response, self.spider) == response+ assert self.mw.process_response(request1, response) == responserequest2 = Request(url2)- self.mw.process_request(request2, self.spider)+ self.mw.process_request(request2)actual_cookies = request2.headers.get("Cookie")assert actual_cookies == (b"a=b" if cookies else None)@@ -743,7 +735,7 @@ class TestCookiesMiddleware:input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)- self.mw.process_request(request1, self.spider)+ self.mw.process_request(request1)cookies = request1.headers.get("Cookie")assert cookies == (b"a=b" if cookies1 else None)@@ -752,18 +744,14 @@ class TestCookiesMiddleware:headers={"Location": f"{to_scheme}://a.example"},status=301,)- assert self.mw.process_response(request1, response, self.spider) == response+ assert self.mw.process_response(request1, response) == response- request2 = self.redirect_middleware.process_response(- request1,- response,- self.spider,- )+ request2 = self.redirect_middleware.process_response(request1, response)assert isinstance(request2, Request)cookies = request2.headers.get("Cookie")assert cookies == (b"a=b" if cookies2 else None)- self.mw.process_request(request2, self.spider)+ self.mw.process_request(request2)cookies = request2.headers.get("Cookie")assert cookies == (b"a=b" if cookies3 else None)diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.pyindex 5716e3631..e697de9b7 100644--- a/tests/test_downloadermiddleware_defaultheaders.py+++ b/tests/test_downloadermiddleware_defaultheaders.py@@ -6,28 +6,27 @@ from scrapy.utils.test import get_crawlerclass TestDefaultHeadersMiddleware:- def get_defaults_spider_mw(self):+ def get_defaults_mw(self):crawler = get_crawler(Spider)- spider = crawler._create_spider("foo")defaults = {to_bytes(k): [to_bytes(v)]for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()}- return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)+ return defaults, DefaultHeadersMiddleware.from_crawler(crawler)def test_process_request(self):- defaults, spider, mw = self.get_defaults_spider_mw()+ defaults, mw = self.get_defaults_mw()req = Request("http://www.scrapytest.org")- mw.process_request(req, spider)+ mw.process_request(req)assert req.headers == defaultsdef test_update_headers(self):- defaults, spider, mw = self.get_defaults_spider_mw()+ defaults, mw = self.get_defaults_mw()headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}req = Request("http://www.scrapytest.org", headers=headers)assert req.headers == bytes_headers- mw.process_request(req, spider)+ mw.process_request(req)defaults.update(bytes_headers)assert req.headers == defaultsdiff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.pyindex 31323c8fa..3707cee18 100644--- a/tests/test_downloadermiddleware_downloadtimeout.py+++ b/tests/test_downloadermiddleware_downloadtimeout.py@@ -14,20 +14,20 @@ class TestDownloadTimeoutMiddleware:def test_default_download_timeout(self):req, spider, mw = self.get_request_spider_mw()mw.spider_opened(spider)- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta.get("download_timeout") == 180def test_string_download_timeout(self):req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})mw.spider_opened(spider)- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta.get("download_timeout") == 20.1def test_spider_has_download_timeout(self):req, spider, mw = self.get_request_spider_mw()spider.download_timeout = 2mw.spider_opened(spider)- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta.get("download_timeout") == 2def test_request_has_download_timeout(self):@@ -35,5 +35,5 @@ class TestDownloadTimeoutMiddleware:spider.download_timeout = 2mw.spider_opened(spider)req.meta["download_timeout"] = 1- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta.get("download_timeout") == 1diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex 9154e1850..522a3002f 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -36,48 +36,48 @@ class TestHttpAuthMiddlewareLegacy:class TestHttpAuthMiddleware:def setup_method(self):self.mw = HttpAuthMiddleware()- self.spider = DomainSpider("foo")- self.mw.spider_opened(self.spider)+ spider = DomainSpider("foo")+ self.mw.spider_opened(spider)def teardown_method(self):del self.mwdef test_no_auth(self):req = Request("http://example-noauth.com/")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert "Authorization" not in req.headersdef test_auth_domain(self):req = Request("http://example.com/")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers["Authorization"] == basic_auth_header("foo", "bar")def test_auth_subdomain(self):req = Request("http://foo.example.com/")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers["Authorization"] == basic_auth_header("foo", "bar")def test_auth_already_set(self):req = Request("http://example.com/", headers={"Authorization": "Digest 123"})- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers["Authorization"] == b"Digest 123"class TestHttpAuthAnyMiddleware:def setup_method(self):self.mw = HttpAuthMiddleware()- self.spider = AnyDomainSpider("foo")- self.mw.spider_opened(self.spider)+ spider = AnyDomainSpider("foo")+ self.mw.spider_opened(spider)def teardown_method(self):del self.mwdef test_auth(self):req = Request("http://example.com/")- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers["Authorization"] == basic_auth_header("foo", "bar")def test_auth_already_set(self):req = Request("http://example.com/", headers={"Authorization": "Digest 123"})- assert self.mw.process_request(req, self.spider) is None+ assert self.mw.process_request(req) is Noneassert req.headers["Authorization"] == b"Digest 123"diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex e60d18375..e94591038 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -1,28 +1,36 @@+from __future__ import annotations+import email.utilsimport shutilimport tempfileimport timefrom contextlib import contextmanager+from typing import TYPE_CHECKING, Anyimport pytestfrom scrapy.downloadermiddlewares.httpcache import HttpCacheMiddlewarefrom scrapy.exceptions import IgnoreRequestfrom scrapy.http import HtmlResponse, Request, Response-from scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler+if TYPE_CHECKING:+ from collections.abc import Generator++ from scrapy.crawler import Crawler+class TestBase:"""Base class with common setup and helper methods."""+ policy_class: str+ storage_class: str+def setup_method(self):self.yesterday = email.utils.formatdate(time.time() - 86400)self.today = email.utils.formatdate()self.tomorrow = email.utils.formatdate(time.time() + 86400)- self.crawler = get_crawler(Spider)- self.spider = self.crawler._create_spider("example.com")self.tmpdir = tempfile.mkdtemp()self.request = Request("http://www.example.com", headers={"User-Agent": "test"})self.response = Response(@@ -31,13 +39,11 @@ class TestBase:body=b"test body",status=202,)- self.crawler.stats.open_spider()def teardown_method(self):- self.crawler.stats.close_spider()shutil.rmtree(self.tmpdir)- def _get_settings(self, **new_settings):+ def _get_settings(self, **new_settings: Any) -> dict[str, Any]:settings = {"HTTPCACHE_ENABLED": True,"HTTPCACHE_DIR": self.tmpdir,@@ -47,27 +53,35 @@ class TestBase:"HTTPCACHE_STORAGE": self.storage_class,}settings.update(new_settings)- return Settings(settings)+ return settings@contextmanager- def _storage(self, **new_settings):- with self._middleware(**new_settings) as mw:- yield mw.storage+ def _get_crawler(self, **new_settings: Any) -> Generator[Crawler]:+ settings = self._get_settings(**new_settings)+ crawler = get_crawler(Spider, settings)+ crawler.spider = crawler._create_spider("example.com")+ assert crawler.stats+ crawler.stats.open_spider()+ try:+ yield crawler+ finally:+ crawler.stats.close_spider()@contextmanager- def _policy(self, **new_settings):+ def _storage(self, **new_settings: Any):with self._middleware(**new_settings) as mw:- yield mw.policy+ yield mw.storage, mw.crawler@contextmanager- def _middleware(self, **new_settings):- settings = self._get_settings(**new_settings)- mw = HttpCacheMiddleware(settings, self.crawler.stats)- mw.spider_opened(self.spider)- try:- yield mw- finally:- mw.spider_closed(self.spider)+ def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]:+ with self._get_crawler(**new_settings) as crawler:+ assert crawler.spider+ mw = HttpCacheMiddleware.from_crawler(crawler)+ mw.spider_opened(crawler.spider)+ try:+ yield mw+ finally:+ mw.spider_closed(crawler.spider)def assertEqualResponse(self, response1, response2):assert response1.url == response2.url@@ -94,37 +108,37 @@ class StorageTestMixin:"""Mixin containing storage-specific test methods."""def test_storage(self):- with self._storage() as storage:+ with self._storage() as (storage, crawler):request2 = self.request.copy()- assert storage.retrieve_response(self.spider, request2) is None+ assert storage.retrieve_response(crawler.spider, request2) is None- storage.store_response(self.spider, self.request, self.response)- response2 = storage.retrieve_response(self.spider, request2)+ storage.store_response(crawler.spider, self.request, self.response)+ response2 = storage.retrieve_response(crawler.spider, request2)assert isinstance(response2, HtmlResponse) # content-type headerself.assertEqualResponse(self.response, response2)time.sleep(2) # wait for cache to expire- assert storage.retrieve_response(self.spider, request2) is None+ assert storage.retrieve_response(crawler.spider, request2) is Nonedef test_storage_never_expire(self):- with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage:- assert storage.retrieve_response(self.spider, self.request) is None- storage.store_response(self.spider, self.request, self.response)+ with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as (storage, crawler):+ assert storage.retrieve_response(crawler.spider, self.request) is None+ storage.store_response(crawler.spider, self.request, self.response)time.sleep(0.5) # give the chance to expire- assert storage.retrieve_response(self.spider, self.request)+ assert storage.retrieve_response(crawler.spider, self.request)def test_storage_no_content_type_header(self):"""Test that the response body is used to get the right response classeven if there is no Content-Type header"""- with self._storage() as storage:- assert storage.retrieve_response(self.spider, self.request) is None+ with self._storage() as (storage, crawler):+ assert storage.retrieve_response(crawler.spider, self.request) is Noneresponse = Response("http://www.example.com",body=b"<!DOCTYPE html>\n<title>.</title>",status=202,)- storage.store_response(self.spider, self.request, response)- cached_response = storage.retrieve_response(self.spider, self.request)+ storage.store_response(crawler.spider, self.request, response)+ cached_response = storage.retrieve_response(crawler.spider, self.request)assert isinstance(cached_response, HtmlResponse)self.assertEqualResponse(response, cached_response)@@ -135,15 +149,15 @@ class PolicyTestMixin:def test_dont_cache(self):with self._middleware() as mw:self.request.meta["dont_cache"] = True- mw.process_response(self.request, self.response, self.spider)- assert mw.storage.retrieve_response(self.spider, self.request) is None+ mw.process_response(self.request, self.response)+ assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is Nonewith self._middleware() as mw:self.request.meta["dont_cache"] = False- mw.process_response(self.request, self.response, self.spider)+ mw.process_response(self.request, self.response)if mw.policy.should_cache_response(self.response, self.request):assert isinstance(- mw.storage.retrieve_response(self.spider, self.request),+ mw.storage.retrieve_response(mw.crawler.spider, self.request),self.response.__class__,)@@ -153,9 +167,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):def test_middleware(self):with self._middleware() as mw:- assert mw.process_request(self.request, self.spider) is None- mw.process_response(self.request, self.response, self.spider)- response = mw.process_request(self.request, self.spider)+ assert mw.process_request(self.request) is None+ mw.process_response(self.request, self.response)+ response = mw.process_request(self.request)assert isinstance(response, HtmlResponse)self.assertEqualResponse(self.response, response)assert "cached" in response.flags@@ -164,9 +178,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):with self._middleware() as mw:req = Request("http://host.com/path")res = Response("http://host2.net/test.html")- assert mw.process_request(req, self.spider) is None- mw.process_response(req, res, self.spider)- cached = mw.process_request(req, self.spider)+ assert mw.process_request(req) is None+ mw.process_response(req, res)+ cached = mw.process_request(req)assert isinstance(cached, Response)self.assertEqualResponse(res, cached)assert "cached" in cached.flags@@ -174,9 +188,9 @@ class DummyPolicyTestMixin(PolicyTestMixin):def test_middleware_ignore_missing(self):with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:with pytest.raises(IgnoreRequest):- mw.process_request(self.request, self.spider)- mw.process_response(self.request, self.response, self.spider)- response = mw.process_request(self.request, self.spider)+ mw.process_request(self.request)+ mw.process_response(self.request, self.response)+ response = mw.process_request(self.request)assert isinstance(response, HtmlResponse)self.assertEqualResponse(self.response, response)assert "cached" in response.flags@@ -185,10 +199,10 @@ class DummyPolicyTestMixin(PolicyTestMixin):# http responses are cached by defaultreq, res = Request("http://test.com/"), Response("http://test.com/")with self._middleware() as mw:- assert mw.process_request(req, self.spider) is None- mw.process_response(req, res, self.spider)+ assert mw.process_request(req) is None+ mw.process_response(req, res)- cached = mw.process_request(req, self.spider)+ cached = mw.process_request(req)assert isinstance(cached, Response), type(cached)self.assertEqualResponse(res, cached)assert "cached" in cached.flags@@ -196,19 +210,19 @@ class DummyPolicyTestMixin(PolicyTestMixin):# file response is not cached by defaultreq, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt")with self._middleware() as mw:- assert mw.process_request(req, self.spider) is None- mw.process_response(req, res, self.spider)+ assert mw.process_request(req) is None+ mw.process_response(req, res)- assert mw.storage.retrieve_response(self.spider, req) is None- assert mw.process_request(req, self.spider) is None+ assert mw.storage.retrieve_response(mw.crawler.spider, req) is None+ assert mw.process_request(req) is None# s3 scheme response is cached by defaultreq, res = Request("s3://bucket/key"), Response("http://bucket/key")with self._middleware() as mw:- assert mw.process_request(req, self.spider) is None- mw.process_response(req, res, self.spider)+ assert mw.process_request(req) is None+ mw.process_response(req, res)- cached = mw.process_request(req, self.spider)+ cached = mw.process_request(req)assert isinstance(cached, Response), type(cached)self.assertEqualResponse(res, cached)assert "cached" in cached.flags@@ -216,25 +230,25 @@ class DummyPolicyTestMixin(PolicyTestMixin):# ignore s3 schemereq, res = Request("s3://bucket/key2"), Response("http://bucket/key2")with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:- assert mw.process_request(req, self.spider) is None- mw.process_response(req, res, self.spider)+ assert mw.process_request(req) is None+ mw.process_response(req, res)- assert mw.storage.retrieve_response(self.spider, req) is None- assert mw.process_request(req, self.spider) is None+ assert mw.storage.retrieve_response(mw.crawler.spider, req) is None+ assert mw.process_request(req) is Nonedef test_middleware_ignore_http_codes(self):# test response is not cachedwith self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) as mw:- assert mw.process_request(self.request, self.spider) is None- mw.process_response(self.request, self.response, self.spider)+ assert mw.process_request(self.request) is None+ mw.process_response(self.request, self.response)- assert mw.storage.retrieve_response(self.spider, self.request) is None- assert mw.process_request(self.request, self.spider) is None+ assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None+ assert mw.process_request(self.request) is None# test response is cachedwith self._middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) as mw:- mw.process_response(self.request, self.response, self.spider)- response = mw.process_request(self.request, self.spider)+ mw.process_response(self.request, self.response)+ response = mw.process_request(self.request)assert isinstance(response, HtmlResponse)self.assertEqualResponse(self.response, response)assert "cached" in response.flags@@ -243,14 +257,18 @@ class DummyPolicyTestMixin(PolicyTestMixin):class RFC2616PolicyTestMixin(PolicyTestMixin):"""Mixin containing RFC2616 policy specific test methods."""- def _process_requestresponse(self, mw, request, response):+ @staticmethod+ def _process_requestresponse(+ mw: HttpCacheMiddleware, request: Request, response: Response | None+ ) -> Response | Request:result = Nonetry:- result = mw.process_request(request, self.spider)+ result = mw.process_request(request)if result:assert isinstance(result, (Request, Response))return result- result = mw.process_response(request, response, self.spider)+ assert response is not None+ result = mw.process_response(request, response)assert isinstance(result, Response)return resultexcept Exception:@@ -270,11 +288,11 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):# response for a request with no-store must not be cachedres1 = self._process_requestresponse(mw, req1, res0)self.assertEqualResponse(res1, res0)- assert mw.storage.retrieve_response(self.spider, req1) is None+ assert mw.storage.retrieve_response(mw.crawler.spider, req1) is None# Re-do request without no-store and expect it to be cachedres2 = self._process_requestresponse(mw, req0, res0)assert "cached" not in res2.flags- res3 = mw.process_request(req0, self.spider)+ res3 = mw.process_request(req0)assert "cached" in res3.flagsself.assertEqualResponse(res2, res3)# request with no-cache directive must not return cached response@@ -330,7 +348,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):)self.assertEqualResponse(res1, res0)self.assertEqualResponse(res2, res0)- resc = mw.storage.retrieve_response(self.spider, req0)+ resc = mw.storage.retrieve_response(mw.crawler.spider, req0)if shouldcache:self.assertEqualResponse(resc, res1)assert "cached" in res2.flags@@ -354,7 +372,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):)self.assertEqualResponse(res1, res0)self.assertEqualResponse(res2, res0)- resc = mw.storage.retrieve_response(self.spider, req0)+ resc = mw.storage.retrieve_response(mw.crawler.spider, req0)if shouldcache:self.assertEqualResponse(resc, res1)assert "cached" in res2.flags@@ -421,7 +439,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):# validate cached response if request max-age set as 0req1 = req0.replace(headers={"Cache-Control": "max-age=0"})res304 = res0.replace(status=304)- assert mw.process_request(req1, self.spider) is None+ assert mw.process_request(req1) is Noneres3 = self._process_requestresponse(mw, req1, res304)self.assertEqualResponse(res1, res3)assert "cached" in res3.flags@@ -513,14 +531,14 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):self._process_requestresponse(mw, req0, res0)for e in mw.DOWNLOAD_EXCEPTIONS:# Simulate encountering an error on download attempts- assert mw.process_request(req0, self.spider) is None- res1 = mw.process_exception(req0, e("foo"), self.spider)+ assert mw.process_request(req0) is None+ res1 = mw.process_exception(req0, e("foo"))# Use cached response as recoveryassert "cached" in res1.flagsself.assertEqualResponse(res0, res1)# Do not use cached response for unhandled exceptions- mw.process_request(req0, self.spider)- assert mw.process_exception(req0, Exception("foo"), self.spider) is None+ mw.process_request(req0)+ assert mw.process_exception(req0, Exception("foo")) is Nonedef test_ignore_response_cache_controls(self):sampledata = [@@ -578,17 +596,17 @@ class TestDbmStorageWithRFC2616Policy(class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy):dbm_module = "tests.mocks.dummydbm"- def _get_settings(self, **new_settings):+ def _get_settings(self, **new_settings) -> dict[str, Any]:new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module)return super()._get_settings(**new_settings)def test_custom_dbm_module_loaded(self):# make sure our dbm module has been loaded- with self._storage() as storage:+ with self._storage() as (storage, _):assert storage.dbmodule.__name__ == self.dbm_moduleclass TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy):- def _get_settings(self, **new_settings):+ def _get_settings(self, **new_settings) -> dict[str, Any]:new_settings.setdefault("HTTPCACHE_GZIP", True)return super()._get_settings(**new_settings)diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 0e58de1f3..65ba2e7e2 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -70,7 +70,6 @@ def _skip_if_no_zstd() -> None:class TestHttpCompression:def setup_method(self):self.crawler = get_crawler(Spider)- self.spider = self.crawler._create_spider("scrapytest.org")self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)self.crawler.stats.open_spider()@@ -124,7 +123,7 @@ class TestHttpCompression:def test_process_request(self):request = Request("http://scrapytest.org")assert "Accept-Encoding" not in request.headers- self.mw.process_request(request, self.spider)+ self.mw.process_request(request)assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS)def test_process_response_gzip(self):@@ -132,7 +131,7 @@ class TestHttpCompression:request = response.requestassert response.headers["Content-Encoding"] == b"gzip"- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -145,7 +144,7 @@ class TestHttpCompression:response = self._getresponse("br")request = response.requestassert response.headers["Content-Encoding"] == b"br"- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -172,7 +171,7 @@ class TestHttpCompression:propagate=False,level=WARNING,) as log:- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)log.check(("scrapy.downloadermiddlewares.httpcompression",@@ -197,7 +196,7 @@ class TestHttpCompression:response = self._getresponse(check_key)request = response.requestassert response.headers["Content-Encoding"] == b"zstd"- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)if raw_content is None:raw_content = newresponse.bodyelse:@@ -221,7 +220,7 @@ class TestHttpCompression:propagate=False,level=WARNING,) as log:- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)log.check(("scrapy.downloadermiddlewares.httpcompression",@@ -241,7 +240,7 @@ class TestHttpCompression:request = response.requestassert response.headers["Content-Encoding"] == b"deflate"- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -253,7 +252,7 @@ class TestHttpCompression:request = response.requestassert response.headers["Content-Encoding"] == b"deflate"- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -265,7 +264,7 @@ class TestHttpCompression:request = Request("http://scrapytest.org")assert not response.headers.get("Content-Encoding")- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is responseassert newresponse.body.startswith(b"<!DOCTYPE")self.assertStatsEqual("httpcompression/response_count", None)@@ -275,14 +274,14 @@ class TestHttpCompression:response = self._getresponse("gzip")response.headers["Content-Encoding"] = ["uuencode", "gzip"]request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"]def test_multi_compression_single_header(self):response = self._getresponse("gzip-deflate")request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert "Content-Encoding" not in newresponse.headersassert newresponse.body.startswith(b"<!DOCTYPE")@@ -296,7 +295,7 @@ class TestHttpCompression:propagate=False,level=WARNING,) as log:- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)log.check(("scrapy.downloadermiddlewares.httpcompression",@@ -314,7 +313,7 @@ class TestHttpCompression:response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "deflate"]request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert "Content-Encoding" not in newresponse.headersassert newresponse.body.startswith(b"<!DOCTYPE")@@ -323,7 +322,7 @@ class TestHttpCompression:response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]@@ -331,7 +330,7 @@ class TestHttpCompression:response = self._getresponse("gzip-deflate-gzip")response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert "Content-Encoding" not in newresponse.headersassert newresponse.body.startswith(b"<!DOCTYPE")@@ -340,7 +339,7 @@ class TestHttpCompression:response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]@@ -362,7 +361,7 @@ class TestHttpCompression:)request = Request("http://www.example.com/")- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert isinstance(newresponse, HtmlResponse)assert newresponse.body == plainbodyassert newresponse.encoding == resolve_encoding("gb2312")@@ -387,7 +386,7 @@ class TestHttpCompression:)request = Request("http://www.example.com/")- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert isinstance(newresponse, HtmlResponse)assert newresponse.body == plainbodyassert newresponse.encoding == resolve_encoding("gb2312")@@ -410,7 +409,7 @@ class TestHttpCompression:)request = Request("http://www.example.com/index")- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert isinstance(newresponse, respcls)assert newresponse.body == plainbodyassert newresponse.encoding == resolve_encoding("gb2312")@@ -422,7 +421,7 @@ class TestHttpCompression:response.headers["Content-Type"] = "application/gzip"request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -434,7 +433,7 @@ class TestHttpCompression:response.headers["Content-Type"] = "application/octet-stream"request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -446,7 +445,7 @@ class TestHttpCompression:response.headers["Content-Type"] = "binary/octet-stream"request = response.request- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is not responseassert newresponse.body.startswith(b"<!DOCTYPE")assert "Content-Encoding" not in newresponse.headers@@ -494,7 +493,7 @@ class TestHttpCompression:)request = Request("http://www.example.com/")- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert gunzip(newresponse.body) == plainbodyself.assertStatsEqual("httpcompression/response_count", 1)self.assertStatsEqual("httpcompression/response_bytes", 230)@@ -505,7 +504,7 @@ class TestHttpCompression:request = response.requestrequest.method = "HEAD"response = response.replace(body=None)- newresponse = self.mw.process_response(request, response, self.spider)+ newresponse = self.mw.process_response(request, response)assert newresponse is responseassert response.body == b""self.assertStatsEqual("httpcompression/response_count", None)@@ -520,7 +519,7 @@ class TestHttpCompression:response = self._getresponse(f"bomb-{compression_id}")with pytest.raises(IgnoreRequest):- mw.process_response(response.request, response, spider)+ mw.process_response(response.request, response)def test_compression_bomb_setting_br(self):_skip_if_no_br()@@ -549,7 +548,7 @@ class TestHttpCompression:response = self._getresponse(f"bomb-{compression_id}")with pytest.raises(IgnoreRequest):- mw.process_response(response.request, response, spider)+ mw.process_response(response.request, response)@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_br(self):@@ -580,7 +579,7 @@ class TestHttpCompression:response = self._getresponse(f"bomb-{compression_id}")response.meta["download_maxsize"] = 10_000_000with pytest.raises(IgnoreRequest):- mw.process_response(response.request, response, spider)+ mw.process_response(response.request, response)def test_compression_bomb_request_meta_br(self):_skip_if_no_br()@@ -611,7 +610,7 @@ class TestHttpCompression:propagate=False,level=WARNING,) as log:- mw.process_response(response.request, response, spider)+ mw.process_response(response.request, response)log.check(("scrapy.downloadermiddlewares.httpcompression",@@ -655,7 +654,7 @@ class TestHttpCompression:propagate=False,level=WARNING,) as log:- mw.process_response(response.request, response, spider)+ mw.process_response(response.request, response)log.check(("scrapy.downloadermiddlewares.httpcompression",@@ -701,7 +700,7 @@ class TestHttpCompression:propagate=False,level=WARNING,) as log:- mw.process_response(response.request, response, spider)+ mw.process_response(response.request, response)log.check(("scrapy.downloadermiddlewares.httpcompression",diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 31d81e73d..b9d368653 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -8,8 +8,6 @@ from scrapy.http import Requestfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-spider = Spider("foo")-class TestHttpProxyMiddleware:failureException = AssertionError # type: ignore[assignment]@@ -31,7 +29,7 @@ class TestHttpProxyMiddleware:for url in ("http://e.com", "https://e.com", "file:///tmp/a"):req = Request(url)- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.url == urlassert req.meta == {}@@ -47,7 +45,7 @@ class TestHttpProxyMiddleware:("file://tmp/a", None),]:req = Request(url)- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.url == urlassert req.meta.get("proxy") == proxy@@ -55,14 +53,14 @@ class TestHttpProxyMiddleware:os.environ["http_proxy"] = "https://proxy.com"mw = HttpProxyMiddleware()req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta == {"proxy": "https://new.proxy:3128"}def test_proxy_auth(self):os.environ["http_proxy"] = "https://user:pass@proxy:3128"mw = HttpProxyMiddleware()req = Request("http://scrapytest.org")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjpwYXNz"# proxy from request.meta@@ -70,7 +68,7 @@ class TestHttpProxyMiddleware:"http://scrapytest.org",meta={"proxy": "https://username:password@proxy:3128"},)- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert (req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6cGFzc3dvcmQ="@@ -80,14 +78,14 @@ class TestHttpProxyMiddleware:os.environ["http_proxy"] = "https://user:@proxy:3128"mw = HttpProxyMiddleware()req = Request("http://scrapytest.org")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjo="# proxy from request.metareq = Request("http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6"@@ -96,7 +94,7 @@ class TestHttpProxyMiddleware:os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128"mw = HttpProxyMiddleware(auth_encoding="utf-8")req = Request("http://scrapytest.org")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic bcOhbjpwYXNz"@@ -104,14 +102,14 @@ class TestHttpProxyMiddleware:req = Request("http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic w7xzZXI6cGFzcw=="# default latin-1 encodingmw = HttpProxyMiddleware(auth_encoding="latin-1")req = Request("http://scrapytest.org")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic beFuOnBhc3M="@@ -119,7 +117,7 @@ class TestHttpProxyMiddleware:req = Request("http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic /HNlcjpwYXNz"@@ -127,7 +125,7 @@ class TestHttpProxyMiddleware:os.environ["http_proxy"] = "https://proxy.for.http:3128"mw = HttpProxyMiddleware()req = Request("http://noproxy.com", meta={"proxy": None})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert "proxy" in req.metaassert req.meta["proxy"] is None@@ -137,23 +135,23 @@ class TestHttpProxyMiddleware:os.environ["no_proxy"] = "*"req = Request("http://noproxy.com")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert "proxy" not in req.metaos.environ["no_proxy"] = "other.com"req = Request("http://noproxy.com")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert "proxy" in req.metaos.environ["no_proxy"] = "other.com,noproxy.com"req = Request("http://noproxy.com")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert "proxy" not in req.meta# proxy from meta['proxy'] takes precedenceos.environ["no_proxy"] = "*"req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.meta == {"proxy": "http://proxy.com"}def test_no_proxy_invalid_values(self):@@ -166,18 +164,18 @@ class TestHttpProxyMiddleware:def test_add_proxy_without_credentials(self):middleware = HttpProxyMiddleware()request = Request("https://example.com")- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://example.com"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert b"Proxy-Authorization" not in request.headersdef test_add_proxy_with_credentials(self):middleware = HttpProxyMiddleware()request = Request("https://example.com")- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://user1:[email protected]"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"encoded_credentials = middleware._basic_auth_header("user1",@@ -191,9 +189,9 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://example.com"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = None- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] is Noneassert b"Proxy-Authorization" not in request.headers@@ -203,9 +201,9 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = None- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] is Noneassert b"Proxy-Authorization" not in request.headers@@ -218,10 +216,10 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://example.com"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://user1:[email protected]"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"encoded_credentials = middleware._basic_auth_header("user1",@@ -237,9 +235,9 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://user2:[email protected]"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"encoded_credentials = middleware._basic_auth_header("user2",@@ -260,10 +258,10 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://example.com"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"encoded_credentials = middleware._basic_auth_header("user1",@@ -273,7 +271,7 @@ class TestHttpProxyMiddleware:request.meta["proxy"] = "https://example.com"del request.headers[b"Proxy-Authorization"]- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert b"Proxy-Authorization" not in request.headers@@ -283,10 +281,10 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://example.com"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://user1:[email protected]"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.org"encoded_credentials = middleware._basic_auth_header("user1",@@ -300,10 +298,10 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://user1:[email protected]"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.org"encoded_credentials = middleware._basic_auth_header("user1",@@ -313,7 +311,7 @@ class TestHttpProxyMiddleware:# Make sure, indirectly, that _auth_proxy is updated.request.meta["proxy"] = "https://example.com"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert b"Proxy-Authorization" not in request.headers@@ -323,10 +321,10 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://user2:[email protected]"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.org"encoded_credentials = middleware._basic_auth_header("user2",@@ -342,9 +340,9 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://example.org"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta == {"proxy": "https://example.org"}assert b"Proxy-Authorization" not in request.headers@@ -360,10 +358,10 @@ class TestHttpProxyMiddleware:"https://example.com",meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Nonerequest.meta["proxy"] = "https://example.org"del request.headers[b"Proxy-Authorization"]- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta == {"proxy": "https://example.org"}assert b"Proxy-Authorization" not in request.headers@@ -373,7 +371,7 @@ class TestHttpProxyMiddleware:"https://example.com",headers={"Proxy-Authorization": "Basic foo"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert "proxy" not in request.metaassert b"Proxy-Authorization" not in request.headers@@ -384,7 +382,7 @@ class TestHttpProxyMiddleware:headers={"Proxy-Authorization": "Basic foo"},meta={"proxy": None},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] is Noneassert b"Proxy-Authorization" not in request.headers@@ -398,16 +396,16 @@ class TestHttpProxyMiddleware:headers={"Proxy-Authorization": "Basic foo"},meta={"proxy": "https://example.com"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert request.headers["Proxy-Authorization"] == b"Basic foo"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert request.headers["Proxy-Authorization"] == b"Basic foo"request.headers["Proxy-Authorization"] = b"Basic bar"- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert request.headers["Proxy-Authorization"] == b"Basic bar"@@ -422,7 +420,7 @@ class TestHttpProxyMiddleware:headers={"Proxy-Authorization": b"Basic " + encoded_credentials},meta={"proxy": "https://user1:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials@@ -437,7 +435,7 @@ class TestHttpProxyMiddleware:headers={"Proxy-Authorization": b"Basic " + encoded_credentials1},meta={"proxy": "https://user2:[email protected]"},)- assert middleware.process_request(request, spider) is None+ assert middleware.process_request(request) is Noneassert request.meta["proxy"] == "https://example.com"encoded_credentials2 = middleware._basic_auth_header("user2",diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex cace52a27..dc0a31a76 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -28,15 +28,15 @@ UNSET = object())def test_process_request_domain_filtering(allowed_domain, url, allowed):crawler = get_crawler(Spider)- spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])+ crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)request = Request(url)if allowed:- assert mw.process_request(request, spider) is None+ assert mw.process_request(request) is Noneelse:with pytest.raises(IgnoreRequest):- mw.process_request(request, spider)+ mw.process_request(request)@pytest.mark.parametrize(@@ -50,18 +50,18 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):)def test_process_request_dont_filter(value, filtered):crawler = get_crawler(Spider)- spider = crawler._create_spider(name="a", allowed_domains=["a.example"])+ crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)kwargs = {}if value is not UNSET:kwargs["dont_filter"] = valuerequest = Request("https://b.example", **kwargs)if filtered:with pytest.raises(IgnoreRequest):- mw.process_request(request, spider)+ mw.process_request(request)else:- assert mw.process_request(request, spider) is None+ assert mw.process_request(request) is None@pytest.mark.parametrize(@@ -79,9 +79,9 @@ def test_process_request_dont_filter(value, filtered):)def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):crawler = get_crawler(Spider)- spider = crawler._create_spider(name="a", allowed_domains=["a.example"])+ crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)kwargs = {"meta": {}}if allow_offsite is not UNSET:kwargs["meta"]["allow_offsite"] = allow_offsite@@ -90,9 +90,9 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):request = Request("https://b.example", **kwargs)if filtered:with pytest.raises(IgnoreRequest):- mw.process_request(request, spider)+ mw.process_request(request)else:- assert mw.process_request(request, spider) is None+ assert mw.process_request(request) is None@pytest.mark.parametrize(@@ -108,27 +108,27 @@ def test_process_request_no_allowed_domains(value):kwargs = {}if value is not UNSET:kwargs["allowed_domains"] = value- spider = crawler._create_spider(name="a", **kwargs)+ crawler.spider = crawler._create_spider(name="a", **kwargs)mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)request = Request("https://example.com")- assert mw.process_request(request, spider) is None+ assert mw.process_request(request) is Nonedef test_process_request_invalid_domains():crawler = get_crawler(Spider)allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]- spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)+ crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)mw = OffsiteMiddleware.from_crawler(crawler)with warnings.catch_warnings():warnings.simplefilter("ignore", UserWarning)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)request = Request("https://a.example")- assert mw.process_request(request, spider) is None+ assert mw.process_request(request) is Nonefor letter in ("b", "c"):request = Request(f"https://{letter}.example")with pytest.raises(IgnoreRequest):- mw.process_request(request, spider)+ mw.process_request(request)@pytest.mark.parametrize(@@ -149,15 +149,15 @@ def test_process_request_invalid_domains():)def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):crawler = get_crawler(Spider)- spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])+ crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)request = Request(url)if allowed:- assert mw.request_scheduled(request, spider) is None+ assert mw.request_scheduled(request, crawler.spider) is Noneelse:with pytest.raises(IgnoreRequest):- mw.request_scheduled(request, spider)+ mw.request_scheduled(request, crawler.spider)@pytest.mark.parametrize(@@ -171,18 +171,18 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):)def test_request_scheduled_dont_filter(value, filtered):crawler = get_crawler(Spider)- spider = crawler._create_spider(name="a", allowed_domains=["a.example"])+ crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)kwargs = {}if value is not UNSET:kwargs["dont_filter"] = valuerequest = Request("https://b.example", **kwargs)if filtered:with pytest.raises(IgnoreRequest):- mw.request_scheduled(request, spider)+ mw.request_scheduled(request, crawler.spider)else:- assert mw.request_scheduled(request, spider) is None+ assert mw.request_scheduled(request, crawler.spider) is None@pytest.mark.parametrize(@@ -198,24 +198,24 @@ def test_request_scheduled_no_allowed_domains(value):kwargs = {}if value is not UNSET:kwargs["allowed_domains"] = value- spider = crawler._create_spider(name="a", **kwargs)+ crawler.spider = crawler._create_spider(name="a", **kwargs)mw = OffsiteMiddleware.from_crawler(crawler)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)request = Request("https://example.com")- assert mw.request_scheduled(request, spider) is None+ assert mw.request_scheduled(request, crawler.spider) is Nonedef test_request_scheduled_invalid_domains():crawler = get_crawler(Spider)allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]- spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)+ crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)mw = OffsiteMiddleware.from_crawler(crawler)with warnings.catch_warnings():warnings.simplefilter("ignore", UserWarning)- mw.spider_opened(spider)+ mw.spider_opened(crawler.spider)request = Request("https://a.example")- assert mw.request_scheduled(request, spider) is None+ assert mw.request_scheduled(request, crawler.spider) is Nonefor letter in ("b", "c"):request = Request(f"https://{letter}.example")with pytest.raises(IgnoreRequest):- mw.request_scheduled(request, spider)+ mw.request_scheduled(request, crawler.spider)diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex a47459eda..281bc898a 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -11,6 +11,7 @@ from scrapy.exceptions import IgnoreRequestfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.misc import set_environ+from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler@@ -19,7 +20,7 @@ class Base:def test_priority_adjust(self):req = Request("http://a.com")rsp = self.get_response(req, "http://a.com/redirected")- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert req2.priority > req.prioritydef test_dont_redirect(self):@@ -28,7 +29,7 @@ class Base:req = Request(url, meta={"dont_redirect": True})rsp = self.get_response(req, url2)- r = self.mw.process_response(req, rsp, self.spider)+ r = self.mw.process_response(req, rsp)assert isinstance(r, Response)assert r is rsp@@ -36,7 +37,7 @@ class Base:req = Request(url, meta={"dont_redirect": False})rsp = self.get_response(req, url2)- r = self.mw.process_response(req, rsp, self.spider)+ r = self.mw.process_response(req, rsp)assert isinstance(r, Request)def test_post(self):@@ -50,7 +51,7 @@ class Base:)rsp = self.get_response(req, url2)- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert isinstance(req2, Request)assert req2.url == url2assert req2.method == "GET"@@ -67,29 +68,29 @@ class Base:req = Request("http://scrapytest.org/302")rsp = self.get_response(req, "/redirected")- req = self.mw.process_response(req, rsp, self.spider)+ req = self.mw.process_response(req, rsp)assert isinstance(req, Request)assert "redirect_times" in req.metaassert req.meta["redirect_times"] == 1with pytest.raises(IgnoreRequest):- self.mw.process_response(req, rsp, self.spider)+ self.mw.process_response(req, rsp)def test_ttl(self):self.mw.max_redirect_times = 100req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1})rsp = self.get_response(req, "/a")- req = self.mw.process_response(req, rsp, self.spider)+ req = self.mw.process_response(req, rsp)assert isinstance(req, Request)with pytest.raises(IgnoreRequest):- self.mw.process_response(req, rsp, self.spider)+ self.mw.process_response(req, rsp)def test_redirect_urls(self):req1 = Request("http://scrapytest.org/first")rsp1 = self.get_response(req1, "/redirected")- req2 = self.mw.process_response(req1, rsp1, self.spider)+ req2 = self.mw.process_response(req1, rsp1)rsp2 = self.get_response(req1, "/redirected2")- req3 = self.mw.process_response(req2, rsp2, self.spider)+ req3 = self.mw.process_response(req2, rsp2)assert req2.url == "http://scrapytest.org/redirected"assert req2.meta["redirect_urls"] == ["http://scrapytest.org/first"]@@ -102,9 +103,9 @@ class Base:def test_redirect_reasons(self):req1 = Request("http://scrapytest.org/first")rsp1 = self.get_response(req1, "/redirected1")- req2 = self.mw.process_response(req1, rsp1, self.spider)+ req2 = self.mw.process_response(req1, rsp1)rsp2 = self.get_response(req2, "/redirected2")- req3 = self.mw.process_response(req2, rsp2, self.spider)+ req3 = self.mw.process_response(req2, rsp2)assert req2.meta["redirect_reasons"] == [self.reason]assert req3.meta["redirect_reasons"] == [self.reason, self.reason]@@ -124,7 +125,7 @@ class Base:original_request, "https://example.com/a")internal_redirect_request = self.mw.process_response(- original_request, internal_response, self.spider+ original_request, internal_response)assert isinstance(internal_redirect_request, Request)assert original_request.headers == internal_redirect_request.headers@@ -137,7 +138,7 @@ class Base:)http_response = self.get_response(http_request, "http://example.com/a")http_redirect_request = self.mw.process_response(- http_request, http_response, self.spider+ http_request, http_response)assert isinstance(http_redirect_request, Request)assert http_request.headers == http_redirect_request.headers@@ -148,7 +149,7 @@ class Base:original_request, "https://example.com:443/a")to_explicit_port_redirect_request = self.mw.process_response(- original_request, to_explicit_port_response, self.spider+ original_request, to_explicit_port_response)assert isinstance(to_explicit_port_redirect_request, Request)assert original_request.headers == to_explicit_port_redirect_request.headers@@ -159,7 +160,7 @@ class Base:original_request, "https://example.com/a")to_implicit_port_redirect_request = self.mw.process_response(- original_request, to_implicit_port_response, self.spider+ original_request, to_implicit_port_response)assert isinstance(to_implicit_port_redirect_request, Request)assert original_request.headers == to_implicit_port_redirect_request.headers@@ -171,7 +172,7 @@ class Base:original_request, "https://example.com:8080/a")different_port_redirect_request = self.mw.process_response(- original_request, different_port_response, self.spider+ original_request, different_port_response)assert isinstance(different_port_redirect_request, Request)assert {@@ -184,7 +185,7 @@ class Base:original_request, "https://example.org/a")external_redirect_request = self.mw.process_response(- original_request, external_response, self.spider+ original_request, external_response)assert isinstance(external_redirect_request, Request)assert safe_headers == external_redirect_request.headers.to_unicode_dict()@@ -194,7 +195,7 @@ class Base:# domain remains the same.upgrade_response = self.get_response(http_request, "https://example.com/a")upgrade_redirect_request = self.mw.process_response(- http_request, upgrade_response, self.spider+ http_request, upgrade_response)assert isinstance(upgrade_redirect_request, Request)assert {@@ -215,7 +216,7 @@ class Base:original_request, "http://example.com/a")downgrade_redirect_request = self.mw.process_response(- original_request, downgrade_response, self.spider+ original_request, downgrade_response)assert isinstance(downgrade_redirect_request, Request)assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()@@ -227,36 +228,35 @@ class Base:meta = {"proxy": "https://a:@a.example"}request1 = Request("http://example.com", meta=meta)- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -269,36 +269,35 @@ class Base:meta = {"proxy": "https://a:@a.example"}request1 = Request("http://example.com", meta=meta)- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "/a")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "/a")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -311,36 +310,35 @@ class Base:meta = {"proxy": "https://a:@a.example"}request1 = Request("https://example.com", meta=meta)- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -353,36 +351,35 @@ class Base:meta = {"proxy": "https://a:@a.example"}request1 = Request("https://example.com", meta=meta)- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "/a")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "/a")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -395,36 +392,35 @@ class Base:meta = {"proxy": "https://a:@a.example"}request1 = Request("http://example.com", meta=meta)- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -437,36 +433,35 @@ class Base:meta = {"proxy": "https://a:@a.example"}request1 = Request("https://example.com", meta=meta)- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -482,36 +477,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("http://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -527,36 +521,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("http://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "/a")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "/a")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -572,36 +565,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("https://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -617,36 +609,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("https://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "/a")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "/a")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"assert request3.meta["proxy"] == "https://a.example"- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -663,36 +654,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("http://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="assert request2.meta["_auth_proxy"] == "https://b.example"assert request2.meta["proxy"] == "https://b.example"response2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -708,36 +698,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("http://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic YTo="assert request1.meta["_auth_proxy"] == "https://a.example"assert request1.meta["proxy"] == "https://a.example"response1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.metaresponse2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic YTo="assert request3.meta["_auth_proxy"] == "https://a.example"@@ -753,36 +742,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("http://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert "Proxy-Authorization" not in request1.headersassert "_auth_proxy" not in request1.metaassert "proxy" not in request1.metaresponse1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="assert request2.meta["_auth_proxy"] == "https://b.example"assert request2.meta["proxy"] == "https://b.example"response2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.meta@@ -794,36 +782,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("http://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert "Proxy-Authorization" not in request1.headersassert "_auth_proxy" not in request1.metaassert "proxy" not in request1.metaresponse1 = self.get_response(request1, "https://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.metaresponse2 = self.get_response(request2, "http://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.meta@@ -840,36 +827,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("https://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="assert request1.meta["_auth_proxy"] == "https://b.example"assert request1.meta["proxy"] == "https://b.example"response1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="assert request3.meta["_auth_proxy"] == "https://b.example"@@ -885,36 +871,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("https://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="assert request1.meta["_auth_proxy"] == "https://b.example"assert request1.meta["proxy"] == "https://b.example"response1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.metaresponse2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="assert request3.meta["_auth_proxy"] == "https://b.example"@@ -930,36 +915,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("https://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert "Proxy-Authorization" not in request1.headersassert "_auth_proxy" not in request1.metaassert "proxy" not in request1.metaresponse1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert request2.headers["Proxy-Authorization"] == b"Basic YTo="assert request2.meta["_auth_proxy"] == "https://a.example"assert request2.meta["proxy"] == "https://a.example"response2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.meta@@ -971,36 +955,35 @@ class Base:proxy_mw = HttpProxyMiddleware.from_crawler(crawler)request1 = Request("https://example.com")- spider = None- proxy_mw.process_request(request1, spider)+ proxy_mw.process_request(request1)assert "Proxy-Authorization" not in request1.headersassert "_auth_proxy" not in request1.metaassert "proxy" not in request1.metaresponse1 = self.get_response(request1, "http://example.com")- request2 = redirect_mw.process_response(request1, response1, spider)+ request2 = redirect_mw.process_response(request1, response1)assert isinstance(request2, Request)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.meta- proxy_mw.process_request(request2, spider)+ proxy_mw.process_request(request2)assert "Proxy-Authorization" not in request2.headersassert "_auth_proxy" not in request2.metaassert "proxy" not in request2.metaresponse2 = self.get_response(request2, "https://example.com")- request3 = redirect_mw.process_response(request2, response2, spider)+ request3 = redirect_mw.process_response(request2, response2)assert isinstance(request3, Request)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.metaassert "proxy" not in request3.meta- proxy_mw.process_request(request3, spider)+ proxy_mw.process_request(request3)assert "Proxy-Authorization" not in request3.headersassert "_auth_proxy" not in request3.meta@@ -1012,9 +995,9 @@ class TestRedirectMiddleware(Base.Test):reason = 302def setup_method(self):- self.crawler = get_crawler(Spider)- self.spider = self.crawler._create_spider("foo")- self.mw = self.mwcls.from_crawler(self.crawler)+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ self.mw = self.mwcls.from_crawler(crawler)def get_response(self, request, location, status=302):headers = {"Location": location}@@ -1027,14 +1010,14 @@ class TestRedirectMiddleware(Base.Test):req = Request(url, method=method)rsp = Response(url, headers={"Location": url2}, status=status)- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert isinstance(req2, Request)assert req2.url == url2assert req2.method == method# response without Location header but with status code is 3XX should be ignoreddel rsp.headers["Location"]- assert self.mw.process_response(req, rsp, self.spider) is rsp+ assert self.mw.process_response(req, rsp) is rsp_test("GET")_test("POST")@@ -1054,7 +1037,7 @@ class TestRedirectMiddleware(Base.Test):req = Request(url, method="HEAD")rsp = Response(url, headers={"Location": url2}, status=302)- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert isinstance(req2, Request)assert req2.url == url2assert req2.method == "HEAD"@@ -1066,19 +1049,18 @@ class TestRedirectMiddleware(Base.Test):req = Request(url, method="HEAD")rsp = Response(url, headers={"Location": url2}, status=302)- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert isinstance(req2, Request)assert req2.url == url3assert req2.method == "HEAD"def test_spider_handling(self):- smartspider = self.crawler._create_spider("smarty")- smartspider.handle_httpstatus_list = [404, 301, 302]+ self.mw.crawler.spider.handle_httpstatus_list = [404, 301, 302]url = "http://www.example.com/301"url2 = "http://www.example.com/redirected"req = Request(url)rsp = Response(url, headers={"Location": url2}, status=301)- r = self.mw.process_response(req, rsp, smartspider)+ r = self.mw.process_response(req, rsp)assert r is rspdef test_request_meta_handling(self):@@ -1087,7 +1069,7 @@ class TestRedirectMiddleware(Base.Test):def _test_passthrough(req):rsp = Response(url, headers={"Location": url2}, status=301, request=req)- r = self.mw.process_response(req, rsp, self.spider)+ r = self.mw.process_response(req, rsp)assert r is rsp_test_passthrough(@@ -1103,7 +1085,7 @@ class TestRedirectMiddleware(Base.Test):headers={"Location": latin1_location},status=302,)- req_result = self.mw.process_response(req, resp, self.spider)+ req_result = self.mw.process_response(req, resp)perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o"assert perc_encoded_utf8_url == req_result.url@@ -1115,14 +1097,14 @@ class TestRedirectMiddleware(Base.Test):headers={"Location": utf8_location},status=302,)- req_result = self.mw.process_response(req, resp, self.spider)+ req_result = self.mw.process_response(req, resp)perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o"assert perc_encoded_utf8_url == req_result.urldef test_no_location(self):request = Request("https://example.com")response = Response(request.url, status=302)- assert self.mw.process_response(request, response, self.spider) is response+ assert self.mw.process_response(request, response) is responseSCHEME_PARAMS = ("url", "location", "target")@@ -1166,11 +1148,10 @@ REDIRECT_SCHEME_CASES = (@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES)def test_redirect_schemes(url, location, target):crawler = get_crawler(Spider)- spider = crawler._create_spider("foo")mw = RedirectMiddleware.from_crawler(crawler)request = Request(url)response = Response(url, headers={"Location": location}, status=301)- redirect = mw.process_response(request, response, spider)+ redirect = mw.process_response(request, response)if target is None:assert redirect == responseelse:@@ -1189,7 +1170,6 @@ class TestMetaRefreshMiddleware(Base.Test):def setup_method(self):crawler = get_crawler(Spider)- self.spider = crawler._create_spider("foo")self.mw = self.mwcls.from_crawler(crawler)def _body(self, interval=5, url="http://example.org/newpage"):@@ -1201,7 +1181,7 @@ class TestMetaRefreshMiddleware(Base.Test):def test_meta_refresh(self):req = Request(url="http://example.org")rsp = HtmlResponse(req.url, body=self._body())- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert isinstance(req2, Request)assert req2.url == "http://example.org/newpage"@@ -1211,7 +1191,7 @@ class TestMetaRefreshMiddleware(Base.Test):rsp = HtmlResponse(url="http://example.org", body=self._body(interval=1000), encoding="utf-8")- rsp2 = self.mw.process_response(req, rsp, self.spider)+ rsp2 = self.mw.process_response(req, rsp)assert rsp is rsp2def test_meta_refresh_trough_posted_request(self):@@ -1222,7 +1202,7 @@ class TestMetaRefreshMiddleware(Base.Test):headers={"Content-Type": "text/plain", "Content-length": "4"},)rsp = HtmlResponse(req.url, body=self._body())- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert isinstance(req2, Request)assert req2.url == "http://example.org/newpage"@@ -1242,7 +1222,7 @@ class TestMetaRefreshMiddleware(Base.Test):"""content="0;URL='http://example.org/newpage'"></noscript>""")rsp = HtmlResponse(req.url, body=body.encode())- response = self.mw.process_response(req, rsp, self.spider)+ response = self.mw.process_response(req, rsp)assert isinstance(response, Response)def test_ignore_tags_1_x_list(self):@@ -1256,7 +1236,7 @@ class TestMetaRefreshMiddleware(Base.Test):"""content="0;URL='http://example.org/newpage'"></noscript>""")rsp = HtmlResponse(req.url, body=body.encode())- response = mw.process_response(req, rsp, self.spider)+ response = mw.process_response(req, rsp)assert isinstance(response, Response)@@ -1288,11 +1268,10 @@ class TestMetaRefreshMiddleware(Base.Test):)def test_meta_refresh_schemes(url, location, target):crawler = get_crawler(Spider)- spider = crawler._create_spider("foo")mw = MetaRefreshMiddleware.from_crawler(crawler)request = Request(url)response = HtmlResponse(url, body=meta_refresh_body(location))- redirect = mw.process_response(request, response, spider)+ redirect = mw.process_response(request, response)if target is None:assert redirect == responseelse:diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 10fc88026..affcc79c1 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -19,20 +19,21 @@ from scrapy.exceptions import IgnoreRequestfrom scrapy.http import Request, Responsefrom scrapy.settings.default_settings import RETRY_EXCEPTIONSfrom scrapy.spiders import Spider+from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerclass TestRetry:def setup_method(self):- self.crawler = get_crawler(Spider)- self.spider = self.crawler._create_spider("foo")+ self.crawler = get_crawler(DefaultSpider)+ self.crawler.spider = self.crawler._create_spider()self.mw = RetryMiddleware.from_crawler(self.crawler)self.mw.max_retry_times = 2def test_priority_adjust(self):req = Request("http://www.scrapytest.org/503")rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)- req2 = self.mw.process_response(req, rsp, self.spider)+ req2 = self.mw.process_response(req, rsp)assert req2.priority < req.prioritydef test_404(self):@@ -40,14 +41,14 @@ class TestRetry:rsp = Response("http://www.scrapytest.org/404", body=b"", status=404)# dont retry 404s- assert self.mw.process_response(req, rsp, self.spider) is rsp+ assert self.mw.process_response(req, rsp) is rspdef test_dont_retry(self):req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)# first retry- r = self.mw.process_response(req, rsp, self.spider)+ r = self.mw.process_response(req, rsp)assert r is rsp# Test retry when dont_retry set to False@@ -55,13 +56,13 @@ class TestRetry:rsp = Response("http://www.scrapytest.org/503")# first retry- r = self.mw.process_response(req, rsp, self.spider)+ r = self.mw.process_response(req, rsp)assert r is rspdef test_dont_retry_exc(self):req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})- r = self.mw.process_exception(req, DNSLookupError(), self.spider)+ r = self.mw.process_exception(req, DNSLookupError())assert r is Nonedef test_503(self):@@ -69,17 +70,17 @@ class TestRetry:rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)# first retry- req = self.mw.process_response(req, rsp, self.spider)+ req = self.mw.process_response(req, rsp)assert isinstance(req, Request)assert req.meta["retry_times"] == 1# second retry- req = self.mw.process_response(req, rsp, self.spider)+ req = self.mw.process_response(req, rsp)assert isinstance(req, Request)assert req.meta["retry_times"] == 2# discard it- assert self.mw.process_response(req, rsp, self.spider) is rsp+ assert self.mw.process_response(req, rsp) is rspassert self.crawler.stats.get_value("retry/max_reached") == 1assert (@@ -118,7 +119,8 @@ class TestRetry:settings_dict = {"RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc],}- crawler = get_crawler(Spider, settings_dict=settings_dict)+ crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)+ crawler.spider = crawler._create_spider()mw = RetryMiddleware.from_crawler(crawler)req = Request(f"http://www.scrapytest.org/{exc.__name__}")self._test_retry_exception(req, exc("foo"), mw)@@ -128,65 +130,61 @@ class TestRetry:mw = self.mw# first retry- req = mw.process_exception(req, exception, self.spider)+ req = mw.process_exception(req, exception)assert isinstance(req, Request)assert req.meta["retry_times"] == 1# second retry- req = mw.process_exception(req, exception, self.spider)+ req = mw.process_exception(req, exception)assert isinstance(req, Request)assert req.meta["retry_times"] == 2# discard it- req = mw.process_exception(req, exception, self.spider)+ req = mw.process_exception(req, exception)assert req is Noneclass TestMaxRetryTimes:invalid_url = "http://www.scrapytest.org/invalid_url"- def get_spider_and_middleware(self, settings=None):- crawler = get_crawler(Spider, settings or {})- spider = crawler._create_spider("foo")- middleware = RetryMiddleware.from_crawler(crawler)- return spider, middleware+ def get_middleware(self, settings=None):+ crawler = get_crawler(DefaultSpider, settings or {})+ crawler.spider = crawler._create_spider()+ return RetryMiddleware.from_crawler(crawler)def test_with_settings_zero(self):max_retry_times = 0settings = {"RETRY_TIMES": max_retry_times}- spider, middleware = self.get_spider_and_middleware(settings)+ middleware = self.get_middleware(settings)req = Request(self.invalid_url)self._test_retry(req,DNSLookupError("foo"),max_retry_times,- spider=spider,middleware=middleware,)def test_with_metakey_zero(self):max_retry_times = 0- spider, middleware = self.get_spider_and_middleware()+ middleware = self.get_middleware()meta = {"max_retry_times": max_retry_times}req = Request(self.invalid_url, meta=meta)self._test_retry(req,DNSLookupError("foo"),max_retry_times,- spider=spider,middleware=middleware,)def test_without_metakey(self):max_retry_times = 5settings = {"RETRY_TIMES": max_retry_times}- spider, middleware = self.get_spider_and_middleware(settings)+ middleware = self.get_middleware(settings)req = Request(self.invalid_url)self._test_retry(req,DNSLookupError("foo"),max_retry_times,- spider=spider,middleware=middleware,)@@ -198,20 +196,18 @@ class TestMaxRetryTimes:req2 = Request(self.invalid_url)settings = {"RETRY_TIMES": middleware_max_retry_times}- spider, middleware = self.get_spider_and_middleware(settings)+ middleware = self.get_middleware(settings)self._test_retry(req1,DNSLookupError("foo"),meta_max_retry_times,- spider=spider,middleware=middleware,)self._test_retry(req2,DNSLookupError("foo"),middleware_max_retry_times,- spider=spider,middleware=middleware,)@@ -223,26 +219,24 @@ class TestMaxRetryTimes:req2 = Request(self.invalid_url)settings = {"RETRY_TIMES": middleware_max_retry_times}- spider, middleware = self.get_spider_and_middleware(settings)+ middleware = self.get_middleware(settings)self._test_retry(req1,DNSLookupError("foo"),meta_max_retry_times,- spider=spider,middleware=middleware,)self._test_retry(req2,DNSLookupError("foo"),middleware_max_retry_times,- spider=spider,middleware=middleware,)def test_with_dont_retry(self):max_retry_times = 4- spider, middleware = self.get_spider_and_middleware()+ middleware = self.get_middleware()meta = {"max_retry_times": max_retry_times,"dont_retry": True,@@ -252,7 +246,6 @@ class TestMaxRetryTimes:req,DNSLookupError("foo"),0,- spider=spider,middleware=middleware,)@@ -261,18 +254,16 @@ class TestMaxRetryTimes:req,exception,max_retry_times,- spider=None,middleware=None,):- spider = spider or self.spidermiddleware = middleware or self.mwfor i in range(max_retry_times):- req = middleware.process_exception(req, exception, spider)+ req = middleware.process_exception(req, exception)assert isinstance(req, Request)# discard it- req = middleware.process_exception(req, exception, spider)+ req = middleware.process_exception(req, exception)assert req is Nonediff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 3cf570180..bedf40279 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, DeferredListfrom twisted.python import failurefrom scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware-from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_loggerfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponsefrom scrapy.http.request import NO_CALLBACK@@ -83,10 +82,10 @@ Disallow: /some/randome/page.htmlasync def test_robotstxt_multiple_reqs(self) -> None:middleware = RobotsTxtMiddleware(self._get_successful_crawler())d1 = deferred_from_coro(- middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type]+ middleware.process_request(Request("http://site.local/allowed1")))d2 = deferred_from_coro(- middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type]+ middleware.process_request(Request("http://site.local/allowed2")))await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True))@@ -94,8 +93,8 @@ Disallow: /some/randome/page.html@deferred_f_from_coro_fasync def test_robotstxt_multiple_reqs_asyncio(self) -> None:middleware = RobotsTxtMiddleware(self._get_successful_crawler())- c1 = middleware.process_request(Request("http://site.local/allowed1"), None) # type: ignore[arg-type]- c2 = middleware.process_request(Request("http://site.local/allowed2"), None) # type: ignore[arg-type]+ c1 = middleware.process_request(Request("http://site.local/allowed1"))+ c2 = middleware.process_request(Request("http://site.local/allowed2"))await asyncio.gather(c1, c2)@deferred_f_from_coro_f@@ -164,7 +163,7 @@ Disallow: /some/randome/page.htmlawait self.assertNotIgnored(Request("http://site.local/static/"), middleware)@deferred_f_from_coro_f- async def test_robotstxt_error(self):+ async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None:self.crawler.settings.set("ROBOTSTXT_OBEY", True)err = error.DNSLookupError("Robotstxt address not found")@@ -176,9 +175,8 @@ Disallow: /some/randome/page.htmlself.crawler.engine.download_async.side_effect = return_failuremiddleware = RobotsTxtMiddleware(self.crawler)- middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)- await middleware.process_request(Request("http://site.local"), None)- assert middleware._logerror.called+ await middleware.process_request(Request("http://site.local"))+ assert "DNS lookup failed: Robotstxt address not found" in caplog.text@deferred_f_from_coro_fasync def test_robotstxt_immediate_error(self):@@ -205,10 +203,13 @@ Disallow: /some/randome/page.htmlself.crawler.engine.download_async.side_effect = ignore_requestmiddleware = RobotsTxtMiddleware(self.crawler)- mw_module_logger.error = mock.MagicMock()-- await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)- assert not mw_module_logger.error.called # type: ignore[attr-defined]+ with mock.patch(+ "scrapy.downloadermiddlewares.robotstxt.logger"+ ) as mw_module_logger:+ await self.assertNotIgnored(+ Request("http://site.local/allowed"), middleware+ )+ assert not mw_module_logger.error.calleddef test_robotstxt_user_agent_setting(self):crawler = self._get_successful_crawler()@@ -216,7 +217,7 @@ Disallow: /some/randome/page.htmlcrawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)")middleware = RobotsTxtMiddleware(crawler)rp = mock.MagicMock(return_value=True)- middleware.process_request_2(rp, Request("http://site.local/allowed"), None)+ middleware.process_request_2(rp, Request("http://site.local/allowed"))rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")@deferred_f_from_coro_f@@ -224,34 +225,30 @@ Disallow: /some/randome/page.htmlmiddleware = RobotsTxtMiddleware(self._get_emptybody_crawler())middleware.process_request_2 = mock.MagicMock()- await middleware.process_request(- Request("data:text/plain,Hello World data"), None- )+ await middleware.process_request(Request("data:text/plain,Hello World data"))assert not middleware.process_request_2.calledawait middleware.process_request(- Request("file:///tests/sample_data/test_site/nothinghere.html"), None+ Request("file:///tests/sample_data/test_site/nothinghere.html"))assert not middleware.process_request_2.called- await middleware.process_request(Request("http://site.local/allowed"), None)+ await middleware.process_request(Request("http://site.local/allowed"))assert middleware.process_request_2.calledasync def assertNotIgnored(self, request: Request, middleware: RobotsTxtMiddleware) -> None:- spider = None # not actually usedtry:- await middleware.process_request(request, spider) # type: ignore[arg-type]+ await middleware.process_request(request)except IgnoreRequest:pytest.fail("IgnoreRequest was raised unexpectedly")async def assertIgnored(self, request: Request, middleware: RobotsTxtMiddleware) -> None:- spider = None # not actually usedwith pytest.raises(IgnoreRequest):- await middleware.process_request(request, spider) # type: ignore[arg-type]+ await middleware.process_request(request)def assertRobotsTxtRequested(self, base_url: str) -> None:calls = self.crawler.engine.download_async.call_args_listdiff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 2f1c97a34..67af4264c 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -11,7 +11,6 @@ class MyException(Exception):class TestDownloaderStats:def setup_method(self):self.crawler = get_crawler(Spider)- self.spider = self.crawler._create_spider("scrapytest.org")self.mw = DownloaderStats(self.crawler.stats)self.crawler.stats.open_spider()@@ -25,15 +24,15 @@ class TestDownloaderStats:)def test_process_request(self):- self.mw.process_request(self.req, self.spider)+ self.mw.process_request(self.req)self.assertStatsEqual("downloader/request_count", 1)def test_process_response(self):- self.mw.process_response(self.req, self.res, self.spider)+ self.mw.process_response(self.req, self.res)self.assertStatsEqual("downloader/response_count", 1)def test_process_exception(self):- self.mw.process_exception(self.req, MyException(), self.spider)+ self.mw.process_exception(self.req, MyException())self.assertStatsEqual("downloader/exception_count", 1)self.assertStatsEqual("downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException",diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.pyindex 1497f8c67..60dc2ae7a 100644--- a/tests/test_downloadermiddleware_useragent.py+++ b/tests/test_downloadermiddleware_useragent.py@@ -11,9 +11,9 @@ class TestUserAgentMiddleware:return spider, UserAgentMiddleware.from_crawler(crawler)def test_default_agent(self):- spider, mw = self.get_spider_and_mw("default_useragent")+ _, mw = self.get_spider_and_mw("default_useragent")req = Request("http://scrapytest.org/")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.headers["User-Agent"] == b"default_useragent"def test_remove_agent(self):@@ -22,7 +22,7 @@ class TestUserAgentMiddleware:spider.user_agent = Nonemw.spider_opened(spider)req = Request("http://scrapytest.org/")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.headers.get("User-Agent") is Nonedef test_spider_agent(self):@@ -30,7 +30,7 @@ class TestUserAgentMiddleware:spider.user_agent = "spider_useragent"mw.spider_opened(spider)req = Request("http://scrapytest.org/")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.headers["User-Agent"] == b"spider_useragent"def test_header_agent(self):@@ -40,7 +40,7 @@ class TestUserAgentMiddleware:req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"})- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert req.headers["User-Agent"] == b"header_useragent"def test_no_agent(self):@@ -48,5 +48,5 @@ class TestUserAgentMiddleware:spider.user_agent = Nonemw.spider_opened(spider)req = Request("http://scrapytest.org/")- assert mw.process_request(req, spider) is None+ assert mw.process_request(req) is Noneassert "User-Agent" not in req.headersdiff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex 6c3008b25..f3d619d3c 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -20,7 +20,7 @@ class M1:def close_spider(self, spider):pass- def process(self, response, request, spider):+ def process(self, response, request):pass@@ -33,7 +33,7 @@ class M2:class M3:- def process(self, response, request, spider):+ def process(self, response, request):passdiff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex de0f0d895..e2c1668d8 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -11,18 +11,18 @@ OVERRIDDEN_URL = "https://example.org"class ProcessResponseMiddleware:- def process_response(self, request, response, spider):+ def process_response(self, request, response):return response.replace(request=Request(OVERRIDDEN_URL))class RaiseExceptionRequestMiddleware:- def process_request(self, request, spider):+ def process_request(self, request):1 / 0return requestclass CatchExceptionOverrideRequestMiddleware:- def process_exception(self, request, exception, spider):+ def process_exception(self, request, exception):return Response(url="http://localhost/",body=b"Caught " + exception.__class__.__name__.encode("utf-8"),@@ -31,7 +31,7 @@ class CatchExceptionOverrideRequestMiddleware:class CatchExceptionDoNotOverrideRequestMiddleware:- def process_exception(self, request, exception, spider):+ def process_exception(self, request, exception):return Response(url="http://localhost/",body=b"Caught " + exception.__class__.__name__.encode("utf-8"),@@ -46,10 +46,17 @@ class AlternativeCallbacksSpider(SingleRequestSpider):class AlternativeCallbacksMiddleware:- def process_response(self, request, response, spider):+ def __init__(self, crawler):+ self.crawler = crawler++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)++ def process_response(self, request, response):new_request = request.replace(url=OVERRIDDEN_URL,- callback=spider.alt_callback,+ callback=self.crawler.spider.alt_callback,cb_kwargs={"foo": "bar"},)return response.replace(request=new_request)diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 0f31af8ad..67ad4ba1e 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -12,11 +12,11 @@ class InjectArgumentsDownloaderMiddleware:Make sure downloader middlewares are able to update the keyword arguments"""- def process_request(self, request, spider):+ def process_request(self, request):if request.callback.__name__ == "parse_downloader_mw":request.cb_kwargs["from_process_request"] = True- def process_response(self, request, response, spider):+ def process_response(self, request, response):if request.callback.__name__ == "parse_downloader_mw":request.cb_kwargs["from_process_response"] = Truereturn responsediff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 658351999..68fcad98c 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -31,15 +31,13 @@ from scrapy.spidermiddlewares.referer import (StrictOriginWhenCrossOriginPolicy,UnsafeUrlPolicy,)-from scrapy.spiders import Spider+from scrapy.utils.spider import DefaultSpider+from scrapy.utils.test import get_crawlerif TYPE_CHECKING:from collections.abc import Sequence--def spider() -> Spider:- return Spider("foo")+ from scrapy.crawler import Crawlerclass TestRefererMiddleware:@@ -1011,20 +1009,24 @@ class TestReferrerOnRedirect(TestRefererMiddleware):]@pytest.fixture- def referrermw(self) -> RefererMiddleware:- settings = Settings(self.settings)- return RefererMiddleware(settings)+ def crawler(self) -> Crawler:+ crawler = get_crawler(DefaultSpider, self.settings)+ crawler.spider = crawler._create_spider()+ return crawler@pytest.fixture- def redirectmw(self) -> RedirectMiddleware:- settings = Settings(self.settings)- return RedirectMiddleware(settings)+ def referrermw(self, crawler: Crawler) -> RefererMiddleware:+ return RefererMiddleware.from_crawler(crawler)++ @pytest.fixture+ def redirectmw(self, crawler: Crawler) -> RedirectMiddleware:+ return RedirectMiddleware.from_crawler(crawler)def test( # type: ignore[override]self,+ crawler: Crawler,referrermw: RefererMiddleware,redirectmw: RedirectMiddleware,- spider: Spider,) -> None:for (parent,@@ -1044,9 +1046,10 @@ class TestReferrerOnRedirect(TestRefererMiddleware):request.url, headers={"Location": url}, status=status)request = cast(- "Request", redirectmw.process_response(request, response, spider)+ "Request", redirectmw.process_response(request, response))- referrermw.request_scheduled(request, spider)+ assert crawler.spider+ referrermw.request_scheduled(request, crawler.spider)assert isinstance(request, Request)assert request.headers.get("Referer") == final_referrer