Scrapy, a fast high-level web crawling & scraping framework for Python.
Full typing for scrapy/downloadermiddlewares (#6129)
Andrey Rakhmatullin7c27c22a987b
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex 04ae719de..0e757e4be 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -1,10 +1,20 @@+from __future__ import annotations+import loggingimport re+from typing import TYPE_CHECKING, Unionfrom w3lib import html+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured-from scrapy.http import HtmlResponse+from scrapy.http import HtmlResponse, Response+from scrapy.settings import BaseSettings++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selflogger = logging.getLogger(__name__)@@ -15,7 +25,7 @@ class AjaxCrawlMiddleware:For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started."""- def __init__(self, settings):+ def __init__(self, settings: BaseSettings):if not settings.getbool("AJAXCRAWL_ENABLED"):raise NotConfigured@@ -23,13 +33,15 @@ class AjaxCrawlMiddleware:# middleware parses first 4k. 4k turns out to be insufficient# for this middleware, and parsing 100k could be slow.# We use something in between (32K) by default.- self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768)+ self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings)- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if not isinstance(response, HtmlResponse) or response.status != 200:return response@@ -54,7 +66,7 @@ class AjaxCrawlMiddleware:ajax_crawl_request.meta["ajax_crawlable"] = Truereturn ajax_crawl_request- def _has_ajax_crawlable_variant(self, response):+ def _has_ajax_crawlable_variant(self, response: Response) -> bool:"""Return True if a page without hash fragment could be "AJAX crawlable"according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.@@ -64,12 +76,12 @@ class AjaxCrawlMiddleware:# XXX: move it to w3lib?-_ajax_crawlable_re = re.compile(+_ajax_crawlable_re: re.Pattern[str] = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')-def _has_ajaxcrawlable_meta(text):+def _has_ajaxcrawlable_meta(text: str) -> bool:""">>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')Truediff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.pyindex 6495157d7..85781efd6 100644--- a/scrapy/downloadermiddlewares/cookies.py+++ b/scrapy/downloadermiddlewares/cookies.py@@ -1,21 +1,41 @@+from __future__ import annotations+import loggingfrom collections import defaultdict+from http.cookiejar import Cookie+from typing import (+ TYPE_CHECKING,+ Any,+ DefaultDict,+ Dict,+ Iterable,+ Optional,+ Sequence,+ Union,+)from tldextract import TLDExtract+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Responsefrom scrapy.http.cookies import CookieJarfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_unicode+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++logger = logging.getLogger(__name__)_split_domain = TLDExtract(include_psl_private_domains=True)-def _is_public_domain(domain):+def _is_public_domain(domain: str) -> bool:parts = _split_domain(domain)return not parts.domain@@ -23,23 +43,27 @@ def _is_public_domain(domain):class CookiesMiddleware:"""This middleware enables working with sites that need cookies"""- def __init__(self, debug=False):- self.jars = defaultdict(CookieJar)- self.debug = debug+ def __init__(self, debug: bool = False):+ self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar)+ self.debug: bool = debug@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("COOKIES_ENABLED"):raise NotConfiguredreturn cls(crawler.settings.getbool("COOKIES_DEBUG"))- def _process_cookies(self, cookies, *, jar, request):+ def _process_cookies(+ self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request+ ) -> None:for cookie in cookies:cookie_domain = cookie.domainif cookie_domain.startswith("."):cookie_domain = cookie_domain[1:]- request_domain = urlparse_cached(request).hostname.lower()+ hostname = urlparse_cached(request).hostname+ assert hostname is not None+ request_domain = hostname.lower()if cookie_domain and _is_public_domain(cookie_domain):if cookie_domain != request_domain:@@ -48,9 +72,11 @@ class CookiesMiddleware:jar.set_cookie_if_ok(cookie, request)- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:if request.meta.get("dont_merge_cookies", False):- return+ return Nonecookiejarkey = request.meta.get("cookiejar")jar = self.jars[cookiejarkey]@@ -61,8 +87,11 @@ class CookiesMiddleware:request.headers.pop("Cookie", None)jar.add_cookie_header(request)self._debug_cookie(request, spider)+ return None- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if request.meta.get("dont_merge_cookies", False):return response@@ -76,7 +105,7 @@ class CookiesMiddleware:return response- def _debug_cookie(self, request, spider):+ def _debug_cookie(self, request: Request, spider: Spider) -> None:if self.debug:cl = [to_unicode(c, errors="replace")@@ -87,7 +116,7 @@ class CookiesMiddleware:msg = f"Sending cookies to: {request}\n{cookies}"logger.debug(msg, extra={"spider": spider})- def _debug_set_cookie(self, response, spider):+ def _debug_set_cookie(self, response: Response, spider: Spider) -> None:if self.debug:cl = [to_unicode(c, errors="replace")@@ -98,7 +127,7 @@ class CookiesMiddleware:msg = f"Received cookies from: {response}\n{cookies}"logger.debug(msg, extra={"spider": spider})- def _format_cookie(self, cookie, request):+ def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:"""Given a dict consisting of cookie components, return its string representation.Decode from bytes if necessary.@@ -109,7 +138,7 @@ class CookiesMiddleware:if key in ("name", "value"):msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"logger.warning(msg)- return+ return Nonecontinueif isinstance(cookie[key], (bool, float, int, str)):decoded[key] = str(cookie[key])@@ -129,12 +158,15 @@ class CookiesMiddleware:cookie_str += f"; {key.capitalize()}={value}"return cookie_str- def _get_request_cookies(self, jar, request):+ def _get_request_cookies(+ self, jar: CookieJar, request: Request+ ) -> Sequence[Cookie]:"""Extract cookies from the Request.cookies attribute"""if not request.cookies:return []+ cookies: Iterable[Dict[str, Any]]if isinstance(request.cookies, dict):cookies = ({"name": k, "value": v} for k, v in request.cookies.items())else:diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.pyindex cdacc7368..8aec37cf1 100644--- a/scrapy/downloadermiddlewares/defaultheaders.py+++ b/scrapy/downloadermiddlewares/defaultheaders.py@@ -3,19 +3,32 @@ DefaultHeaders downloader middlewareSee documentation in docs/topics/downloader-middleware.rst"""+from __future__ import annotations+from typing import TYPE_CHECKING, Iterable, Tuple, Union++from scrapy import Request, Spider+from scrapy.crawler import Crawler+from scrapy.http import Responsefrom scrapy.utils.python import without_none_values+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+class DefaultHeadersMiddleware:- def __init__(self, headers):- self._headers = headers+ def __init__(self, headers: Iterable[Tuple[str, str]]):+ self._headers: Iterable[Tuple[str, str]] = headers@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])return cls(headers.items())- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:for k, v in self._headers:request.headers.setdefault(k, v)+ return Nonediff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.pyindex a926ecf56..1c904c05b 100644--- a/scrapy/downloadermiddlewares/downloadtimeout.py+++ b/scrapy/downloadermiddlewares/downloadtimeout.py@@ -3,23 +3,35 @@ Download timeout middlewareSee documentation in docs/topics/downloader-middleware.rst"""+from __future__ import annotations-from scrapy import signals+from typing import TYPE_CHECKING, Union++from scrapy import Request, Spider, signals+from scrapy.crawler import Crawler+from scrapy.http import Response++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selfclass DownloadTimeoutMiddleware:- def __init__(self, timeout=180):- self._timeout = timeout+ def __init__(self, timeout: float = 180):+ self._timeout: float = timeout@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)return o- def spider_opened(self, spider):+ def spider_opened(self, spider: Spider) -> None:self._timeout = getattr(spider, "download_timeout", self._timeout)- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:if self._timeout:request.meta.setdefault("download_timeout", self._timeout)+ return Nonediff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.pyindex 5228db786..63490a37a 100644--- a/scrapy/downloadermiddlewares/httpauth.py+++ b/scrapy/downloadermiddlewares/httpauth.py@@ -4,31 +4,44 @@ HTTP basic auth downloader middlewareSee documentation in docs/topics/downloader-middleware.rst"""+from __future__ import annotations++from typing import TYPE_CHECKING, Union+from w3lib.http import basic_auth_header-from scrapy import signals+from scrapy import Request, Spider, signals+from scrapy.crawler import Crawler+from scrapy.http import Responsefrom scrapy.utils.url import url_is_from_any_domain+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+class HttpAuthMiddleware:"""Set Basic HTTP Authorization header(http_user and http_pass spider class attributes)"""@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:o = cls()crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)return o- def spider_opened(self, spider):+ def spider_opened(self, spider: Spider) -> None:usr = getattr(spider, "http_user", "")pwd = getattr(spider, "http_pass", "")if usr or pwd:self.auth = basic_auth_header(usr, pwd)- self.domain = spider.http_auth_domain+ self.domain = spider.http_auth_domain # type: ignore[attr-defined]- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:auth = getattr(self, "auth", None)if auth and b"Authorization" not in request.headers:if not self.domain or url_is_from_any_domain(request.url, [self.domain]):request.headers[b"Authorization"] = auth+ return Nonediff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex a521cde7a..971473403 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -1,5 +1,7 @@+from __future__ import annotations+from email.utils import formatdate-from typing import Optional, Type, TypeVar+from typing import TYPE_CHECKING, Optional, Unionfrom twisted.internet import deferfrom twisted.internet.error import (@@ -23,7 +25,9 @@ from scrapy.spiders import Spiderfrom scrapy.statscollectors import StatsCollectorfrom scrapy.utils.misc import load_object-HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selfclass HttpCacheMiddleware:@@ -49,9 +53,7 @@ class HttpCacheMiddleware:self.stats = stats@classmethod- def from_crawler(- cls: Type[HttpCacheMiddlewareTV], crawler: Crawler- ) -> HttpCacheMiddlewareTV:+ def from_crawler(cls, crawler: Crawler) -> Self:assert crawler.statso = cls(crawler.settings, crawler.stats)crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)@@ -64,7 +66,9 @@ class HttpCacheMiddleware:def spider_closed(self, spider: Spider) -> None:self.storage.close_spider(spider)- def process_request(self, request: Request, spider: Spider) -> Optional[Response]:+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:if request.meta.get("dont_cache", False):return None@@ -74,7 +78,9 @@ class HttpCacheMiddleware:return None# Look for cached response and check if expired- cachedresponse = self.storage.retrieve_response(spider, request)+ cachedresponse: Optional[Response] = self.storage.retrieve_response(+ spider, request+ )if cachedresponse is None:self.stats.inc_value("httpcache/miss", spider=spider)if self.ignore_missing:@@ -96,7 +102,7 @@ class HttpCacheMiddleware:def process_response(self, request: Request, response: Response, spider: Spider- ) -> Response:+ ) -> Union[Request, Response]:if request.meta.get("dont_cache", False):return response@@ -111,7 +117,7 @@ class HttpCacheMiddleware:response.headers["Date"] = formatdate(usegmt=True)# Do not validate first-hand responses- cachedresponse = request.meta.pop("cached_response", None)+ cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)if cachedresponse is None:self.stats.inc_value("httpcache/firsthand", spider=spider)self._cache_response(spider, response, request, cachedresponse)@@ -127,8 +133,8 @@ class HttpCacheMiddleware:def process_exception(self, request: Request, exception: Exception, spider: Spider- ) -> Optional[Response]:- cachedresponse = request.meta.pop("cached_response", None)+ ) -> Union[Request, Response, None]:+ cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 7b1d3f829..56a58a750 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,12 +1,22 @@+from __future__ import annotations+import ioimport zlib+from typing import TYPE_CHECKING, List, Optional, Union+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes+from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.gz import gunzip-ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]try:import brotli@@ -27,19 +37,24 @@ class HttpCompressionMiddleware:"""This middleware allows compressed (gzip, deflate) traffic to besent/received from web sites"""- def __init__(self, stats=None):+ def __init__(self, stats: Optional[StatsCollector] = None):self.stats = stats@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("COMPRESSION_ENABLED"):raise NotConfiguredreturn cls(stats=crawler.stats)- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))+ return None- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if request.method == "HEAD":return responseif isinstance(response, Response):@@ -70,7 +85,7 @@ class HttpCompressionMiddleware:return response- def _decode(self, body, encoding):+ def _decode(self, body: bytes, encoding: bytes) -> bytes:if encoding == b"gzip" or encoding == b"x-gzip":body = gunzip(body)diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex f74d84b69..335896ac1 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -1,16 +1,30 @@+from __future__ import annotations+import base64+from typing import TYPE_CHECKING, Dict, Optional, Tuple, Unionfrom urllib.parse import unquote, urlunparse-from urllib.request import _parse_proxy, getproxies, proxy_bypass+from urllib.request import ( # type: ignore[attr-defined]+ _parse_proxy,+ getproxies,+ proxy_bypass,+)+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured+from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+class HttpProxyMiddleware:- def __init__(self, auth_encoding="latin-1"):- self.auth_encoding = auth_encoding- self.proxies = {}+ def __init__(self, auth_encoding: Optional[str] = "latin-1"):+ self.auth_encoding: Optional[str] = auth_encoding+ self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {}for type_, url in getproxies().items():try:self.proxies[type_] = self._get_proxy(url, type_)@@ -20,19 +34,19 @@ class HttpProxyMiddleware:continue@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("HTTPPROXY_ENABLED"):raise NotConfigured- auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")+ auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")return cls(auth_encoding)- def _basic_auth_header(self, username, password):+ def _basic_auth_header(self, username: str, password: str) -> bytes:user_pass = to_bytes(f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding)return base64.b64encode(user_pass)- def _get_proxy(self, url, orig_type):+ def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]:proxy_type, user, password, hostport = _parse_proxy(url)proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))@@ -43,7 +57,9 @@ class HttpProxyMiddleware:return creds, proxy_url- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:creds, proxy_url = None, Noneif "proxy" in request.meta:if request.meta["proxy"] is not None:@@ -54,13 +70,16 @@ class HttpProxyMiddleware:if (# 'no_proxy' is only supported by http schemesscheme not in ("http", "https")- or not proxy_bypass(parsed.hostname)+ or (parsed.hostname and not proxy_bypass(parsed.hostname))) and scheme in self.proxies:creds, proxy_url = self.proxies[scheme]self._set_proxy_and_creds(request, proxy_url, creds)+ return None- def _set_proxy_and_creds(self, request, proxy_url, creds):+ def _set_proxy_and_creds(+ self, request: Request, proxy_url: Optional[str], creds: Optional[bytes]+ ) -> None:if proxy_url:request.meta["proxy"] = proxy_urlelif request.meta.get("proxy") is not None:diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 65f1d2224..814b1a561 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -1,17 +1,29 @@+from __future__ import annotations+import logging+from typing import TYPE_CHECKING, Any, List, Union, castfrom urllib.parse import urljoin, urlparsefrom w3lib.url import safe_url_string+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfigured-from scrapy.http import HtmlResponse+from scrapy.http import HtmlResponse, Response+from scrapy.settings import BaseSettingsfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.response import get_meta_refresh+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+logger = logging.getLogger(__name__)-def _build_redirect_request(source_request, *, url, **kwargs):+def _build_redirect_request(+ source_request: Request, *, url: str, **kwargs: Any+) -> Request:redirect_request = source_request.replace(url=url,**kwargs,@@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs):class BaseRedirectMiddleware:- enabled_setting = "REDIRECT_ENABLED"+ enabled_setting: str = "REDIRECT_ENABLED"- def __init__(self, settings):+ def __init__(self, settings: BaseSettings):if not settings.getbool(self.enabled_setting):raise NotConfigured- self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES")- self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST")+ self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES")+ self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST")@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings)- def _redirect(self, redirected, request, spider, reason):+ def _redirect(+ self, redirected: Request, request: Request, spider: Spider, reason: Any+ ) -> Request:ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)redirects = request.meta.get("redirect_times", 0) + 1@@ -67,7 +81,9 @@ class BaseRedirectMiddleware:)raise IgnoreRequest("max redirections reached")- def _redirect_request_using_get(self, request, redirect_url):+ def _redirect_request_using_get(+ self, request: Request, redirect_url: str+ ) -> Request:redirect_request = _build_redirect_request(request,url=redirect_url,@@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):and meta-refresh html tag."""- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if (request.meta.get("dont_redirect", False)or response.status in getattr(spider, "handle_httpstatus_list", [])@@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):if "Location" not in response.headers or response.status not in allowed_status:return response+ assert response.headers["Location"] is not Nonelocation = safe_url_string(response.headers["Location"])if response.headers["Location"].startswith(b"//"):request_scheme = urlparse(request.url).scheme@@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):class MetaRefreshMiddleware(BaseRedirectMiddleware):enabled_setting = "METAREFRESH_ENABLED"- def __init__(self, settings):+ def __init__(self, settings: BaseSettings):super().__init__(settings)- self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS")- self._maxdelay = settings.getint("METAREFRESH_MAXDELAY")+ self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")+ self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if (request.meta.get("dont_redirect", False)or request.method == "HEAD"@@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):return responseinterval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)- if url and interval < self._maxdelay:+ if url and cast(float, interval) < self._maxdelay:redirected = self._redirect_request_using_get(request, url)return self._redirect(redirected, request, spider, "meta refresh")diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 380623cea..3c494de78 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -9,22 +9,30 @@ RETRY_HTTP_CODES - which HTTP response codes to retryFailed pages are collected on the scraping process and rescheduled at the end,once the spider has finished crawling all regular (non failed) pages."""+from __future__ import annotations+import warningsfrom logging import Logger, getLogger-from typing import Optional, Type, Union+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.http import Responsefrom scrapy.http.request import Request-from scrapy.settings import Settings+from scrapy.settings import BaseSettings, Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import global_object_namefrom scrapy.utils.response import response_status_message+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+retry_logger = getLogger(__name__)-def backwards_compatibility_getattr(self, name):+def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]:if name == "EXCEPTIONS_TO_RETRY":warnings.warn("Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "@@ -54,7 +62,7 @@ def get_retry_request(priority_adjust: Optional[int] = None,logger: Logger = retry_logger,stats_base_key: str = "retry",-):+) -> Optional[Request]:"""Returns a new :class:`~scrapy.Request` object to retry the specifiedrequest, or ``None`` if retries of the specified request have been@@ -134,7 +142,7 @@ def get_retry_request(class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):- def __init__(self, settings):+ def __init__(self, settings: BaseSettings):if not settings.getbool("RETRY_ENABLED"):raise NotConfiguredself.max_retry_times = settings.getint("RETRY_TIMES")@@ -153,10 +161,12 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):)@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings)- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if request.meta.get("dont_retry", False):return responseif response.status in self.retry_http_codes:@@ -164,13 +174,21 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):return self._retry(request, reason, spider) or responsereturn response- def process_exception(self, request, exception, spider):+ def process_exception(+ self, request: Request, exception: Exception, spider: Spider+ ) -> Union[Request, Response, None]:if isinstance(exception, self.exceptions_to_retry) and not request.meta.get("dont_retry", False):return self._retry(request, exception, spider)-- def _retry(self, request, reason, spider):+ return None++ def _retry(+ self,+ request: Request,+ reason: Union[str, Exception, Type[Exception]],+ spider: Spider,+ ) -> Optional[Request]:max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)return get_retry_request(diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 6cab27c5a..6a0ecb7bf 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting."""+from __future__ import annotations+import logging+from typing import TYPE_CHECKING, Any, Dict, Optional, Unionfrom twisted.internet.defer import Deferred, maybeDeferred+from twisted.python.failure import Failure+from scrapy import Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfigured-from scrapy.http import Request+from scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK+from scrapy.robotstxt import RobotParserfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import load_object+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++logger = logging.getLogger(__name__)class RobotsTxtMiddleware:- DOWNLOAD_PRIORITY = 1000+ DOWNLOAD_PRIORITY: int = 1000- def __init__(self, crawler):+ def __init__(self, crawler: Crawler):if not crawler.settings.getbool("ROBOTSTXT_OBEY"):raise NotConfigured- self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy")- self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None)- self.crawler = crawler- self._parsers = {}- self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER"))+ self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")+ self._robotstxt_useragent: Optional[str] = crawler.settings.get(+ "ROBOTSTXT_USER_AGENT", None+ )+ self.crawler: Crawler = crawler+ self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}+ self._parserimpl: RobotParser = load_object(+ crawler.settings.get("ROBOTSTXT_PARSER")+ )# check if parser dependencies are met, this should throw an error otherwise.self._parserimpl.from_crawler(self.crawler, b"")@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler)- def process_request(self, request, spider):+ def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:if request.meta.get("dont_obey_robotstxt"):- return+ return Noneif request.url.startswith("data:") or request.url.startswith("file:"):- return- d = maybeDeferred(self.robot_parser, request, spider)+ return None+ d: Deferred = maybeDeferred(self.robot_parser, request, spider)d.addCallback(self.process_request_2, request, spider)return d- def process_request_2(self, rp, request, spider):+ def process_request_2(+ self, rp: Optional[RobotParser], request: Request, spider: Spider+ ) -> None:if rp is None:return- useragent = self._robotstxt_useragent+ useragent: Union[str, bytes, None] = self._robotstxt_useragentif not useragent:useragent = request.headers.get(b"User-Agent", self._default_useragent)+ assert useragent is not Noneif not rp.allowed(request.url, useragent):logger.debug("Forbidden by robots.txt: %(request)s",{"request": request},extra={"spider": spider},)+ assert self.crawler.statsself.crawler.stats.inc_value("robotstxt/forbidden")raise IgnoreRequest("Forbidden by robots.txt")- def robot_parser(self, request, spider):+ def robot_parser(+ self, request: Request, spider: Spider+ ) -> Union[RobotParser, Deferred, None]:url = urlparse_cached(request)netloc = url.netloc@@ -75,24 +97,27 @@ class RobotsTxtMiddleware:meta={"dont_obey_robotstxt": True},callback=NO_CALLBACK,)+ assert self.crawler.engine+ assert self.crawler.statsdfd = self.crawler.engine.download(robotsreq)dfd.addCallback(self._parse_robots, netloc, spider)dfd.addErrback(self._logerror, robotsreq, spider)dfd.addErrback(self._robots_error, netloc)self.crawler.stats.inc_value("robotstxt/request_count")- if isinstance(self._parsers[netloc], Deferred):- d = Deferred()+ parser = self._parsers[netloc]+ if isinstance(parser, Deferred):+ d: Deferred = Deferred()- def cb(result):+ def cb(result: Any) -> Any:d.callback(result)return result- self._parsers[netloc].addCallback(cb)+ parser.addCallback(cb)return d- return self._parsers[netloc]+ return parser- def _logerror(self, failure, request, spider):+ def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure:if failure.type is not IgnoreRequest:logger.error("Error downloading %(request)s: %(f_exception)s",@@ -102,20 +127,24 @@ class RobotsTxtMiddleware:)return failure- def _parse_robots(self, response, netloc, spider):+ def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None:+ assert self.crawler.statsself.crawler.stats.inc_value("robotstxt/response_count")self.crawler.stats.inc_value(f"robotstxt/response_status_count/{response.status}")rp = self._parserimpl.from_crawler(self.crawler, response.body)rp_dfd = self._parsers[netloc]+ assert isinstance(rp_dfd, Deferred)self._parsers[netloc] = rprp_dfd.callback(rp)- def _robots_error(self, failure, netloc):+ def _robots_error(self, failure: Failure, netloc: str) -> None:if failure.type is not IgnoreRequest:key = f"robotstxt/exception_count/{failure.type}"+ assert self.crawler.statsself.crawler.stats.inc_value(key)rp_dfd = self._parsers[netloc]+ assert isinstance(rp_dfd, Deferred)self._parsers[netloc] = Nonerp_dfd.callback(None)diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex a0f62e262..df30e8ca4 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -1,11 +1,23 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Dict, Union+from twisted.web import http+from scrapy import Request, Spider+from scrapy.crawler import Crawlerfrom scrapy.exceptions import NotConfigured+from scrapy.http import Response+from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.python import global_object_name, to_bytesfrom scrapy.utils.request import request_httprepr+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+-def get_header_size(headers):+def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:size = 0for key, value in headers.items():if isinstance(value, (list, tuple)):@@ -14,30 +26,36 @@ def get_header_size(headers):return size + len(b"\r\n") * (len(headers.keys()) - 1)-def get_status_size(response_status):+def get_status_size(response_status: int) -> int:return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "class DownloaderStats:- def __init__(self, stats):- self.stats = stats+ def __init__(self, stats: StatsCollector):+ self.stats: StatsCollector = stats@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("DOWNLOADER_STATS"):raise NotConfigured+ assert crawler.statsreturn cls(crawler.stats)- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:self.stats.inc_value("downloader/request_count", spider=spider)self.stats.inc_value(f"downloader/request_method_count/{request.method}", spider=spider)reqlen = len(request_httprepr(request))self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)+ return None- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:self.stats.inc_value("downloader/response_count", spider=spider)self.stats.inc_value(f"downloader/response_status_count/{response.status}", spider=spider@@ -52,9 +70,12 @@ class DownloaderStats:self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)return response- def process_exception(self, request, exception, spider):+ def process_exception(+ self, request: Request, exception: Exception, spider: Spider+ ) -> Union[Request, Response, None]:ex_class = global_object_name(exception.__class__)self.stats.inc_value("downloader/exception_count", spider=spider)self.stats.inc_value(f"downloader/exception_type_count/{ex_class}", spider=spider)+ return Nonediff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.pyindex 856a275ab..92f1ec897 100644--- a/scrapy/downloadermiddlewares/useragent.py+++ b/scrapy/downloadermiddlewares/useragent.py@@ -1,23 +1,36 @@"""Set User-Agent header per spider or use a default value from settings"""-from scrapy import signals+from __future__ import annotations++from typing import TYPE_CHECKING, Union++from scrapy import Request, Spider, signals+from scrapy.crawler import Crawler+from scrapy.http import Response++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selfclass UserAgentMiddleware:"""This middleware allows spiders to override the user_agent"""- def __init__(self, user_agent="Scrapy"):+ def __init__(self, user_agent: str = "Scrapy"):self.user_agent = user_agent@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:o = cls(crawler.settings["USER_AGENT"])crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)return o- def spider_opened(self, spider):+ def spider_opened(self, spider: Spider) -> None:self.user_agent = getattr(spider, "user_agent", self.user_agent)- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:if self.user_agent:request.headers.setdefault(b"User-Agent", self.user_agent)+ return Nonediff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex a5329ad51..2595f328e 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -1,8 +1,12 @@import reimport time+from http.cookiejar import Cookiefrom http.cookiejar import CookieJar as _CookieJarfrom http.cookiejar import DefaultCookiePolicy+from typing import Sequence+from scrapy import Request+from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_unicode@@ -24,7 +28,7 @@ class CookieJar:wrsp = WrappedResponse(response)return self.jar.extract_cookies(wrsp, wreq)- def add_cookie_header(self, request):+ def add_cookie_header(self, request: Request) -> None:wreq = WrappedRequest(request)self.policy._now = self.jar._now = int(time.time())@@ -75,7 +79,7 @@ class CookieJar:def set_policy(self, pol):return self.jar.set_policy(pol)- def make_cookies(self, response, request):+ def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]:wreq = WrappedRequest(request)wrsp = WrappedResponse(response)return self.jar.make_cookies(wrsp, wreq)@@ -83,7 +87,7 @@ class CookieJar:def set_cookie(self, cookie):self.jar.set_cookie(cookie)- def set_cookie_if_ok(self, cookie, request):+ def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None:self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 5c5ac4e41..6ea2bfd97 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -1,11 +1,20 @@+from __future__ import annotations+import loggingimport sysfrom abc import ABCMeta, abstractmethod+from typing import TYPE_CHECKING, Unionfrom warnings import warnfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import to_unicode+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy.crawler import Crawler+logger = logging.getLogger(__name__)@@ -31,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):class RobotParser(metaclass=ABCMeta):@classmethod@abstractmethod- def from_crawler(cls, crawler, robotstxt_body):+ def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:"""Parse the content of a robots.txt_ file as bytes. This must be a class method.It must return a new instance of the parser backend.@@ -44,14 +53,14 @@ class RobotParser(metaclass=ABCMeta):pass@abstractmethod- def allowed(self, url, user_agent):+ def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:"""Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.:param url: Absolute URL- :type url: str+ :type url: str or bytes:param user_agent: User agent- :type user_agent: str+ :type user_agent: str or bytes"""passdiff --git a/setup.cfg b/setup.cfgindex db79c5821..d23549f10 100644--- a/setup.cfg+++ b/setup.cfg@@ -16,9 +16,6 @@ follow_imports = skip# FIXME: remove the following sections once the issues are solved-[mypy-scrapy.downloadermiddlewares.httpproxy]-ignore_errors = True-[mypy-scrapy.interfaces]ignore_errors = True