Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Full typing for scrapy/downloadermiddlewares (#6129)

Andrey Rakhmatullin7c27c22a987b
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 04ae719de..0e757e4be 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,10 +1,20 @@
+from __future__ import annotations
+
import logging
import re
+from typing import TYPE_CHECKING, Union
from w3lib import html
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
-from scrapy.http import HtmlResponse
+from scrapy.http import HtmlResponse, Response
+from scrapy.settings import BaseSettings
+
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
logger = logging.getLogger(__name__)
@@ -15,7 +25,7 @@ class AjaxCrawlMiddleware:
For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
"""
- def __init__(self, settings):
+ def __init__(self, settings: BaseSettings):
if not settings.getbool("AJAXCRAWL_ENABLED"):
raise NotConfigured
@@ -23,13 +33,15 @@ class AjaxCrawlMiddleware:
# middleware parses first 4k. 4k turns out to be insufficient
# for this middleware, and parsing 100k could be slow.
# We use something in between (32K) by default.
- self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
+ self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if not isinstance(response, HtmlResponse) or response.status != 200:
return response
@@ -54,7 +66,7 @@ class AjaxCrawlMiddleware:
ajax_crawl_request.meta["ajax_crawlable"] = True
return ajax_crawl_request
- def _has_ajax_crawlable_variant(self, response):
+ def _has_ajax_crawlable_variant(self, response: Response) -> bool:
"""
Return True if a page without hash fragment could be "AJAX crawlable"
according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
@@ -64,12 +76,12 @@ class AjaxCrawlMiddleware:
# XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(
+_ajax_crawlable_re: re.Pattern[str] = re.compile(
r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'
)
-def _has_ajaxcrawlable_meta(text):
+def _has_ajaxcrawlable_meta(text: str) -> bool:
"""
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
True
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 6495157d7..85781efd6 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,21 +1,41 @@
+from __future__ import annotations
+
import logging
from collections import defaultdict
+from http.cookiejar import Cookie
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ DefaultDict,
+ Dict,
+ Iterable,
+ Optional,
+ Sequence,
+ Union,
+)
from tldextract import TLDExtract
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+
logger = logging.getLogger(__name__)
_split_domain = TLDExtract(include_psl_private_domains=True)
-def _is_public_domain(domain):
+def _is_public_domain(domain: str) -> bool:
parts = _split_domain(domain)
return not parts.domain
@@ -23,23 +43,27 @@ def _is_public_domain(domain):
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
- def __init__(self, debug=False):
- self.jars = defaultdict(CookieJar)
- self.debug = debug
+ def __init__(self, debug: bool = False):
+ self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar)
+ self.debug: bool = debug
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COOKIES_ENABLED"):
raise NotConfigured
return cls(crawler.settings.getbool("COOKIES_DEBUG"))
- def _process_cookies(self, cookies, *, jar, request):
+ def _process_cookies(
+ self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
+ ) -> None:
for cookie in cookies:
cookie_domain = cookie.domain
if cookie_domain.startswith("."):
cookie_domain = cookie_domain[1:]
- request_domain = urlparse_cached(request).hostname.lower()
+ hostname = urlparse_cached(request).hostname
+ assert hostname is not None
+ request_domain = hostname.lower()
if cookie_domain and _is_public_domain(cookie_domain):
if cookie_domain != request_domain:
@@ -48,9 +72,11 @@ class CookiesMiddleware:
jar.set_cookie_if_ok(cookie, request)
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
if request.meta.get("dont_merge_cookies", False):
- return
+ return None
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
@@ -61,8 +87,11 @@ class CookiesMiddleware:
request.headers.pop("Cookie", None)
jar.add_cookie_header(request)
self._debug_cookie(request, spider)
+ return None
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if request.meta.get("dont_merge_cookies", False):
return response
@@ -76,7 +105,7 @@ class CookiesMiddleware:
return response
- def _debug_cookie(self, request, spider):
+ def _debug_cookie(self, request: Request, spider: Spider) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@@ -87,7 +116,7 @@ class CookiesMiddleware:
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
- def _debug_set_cookie(self, response, spider):
+ def _debug_set_cookie(self, response: Response, spider: Spider) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@@ -98,7 +127,7 @@ class CookiesMiddleware:
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
- def _format_cookie(self, cookie, request):
+ def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
"""
Given a dict consisting of cookie components, return its string representation.
Decode from bytes if necessary.
@@ -109,7 +138,7 @@ class CookiesMiddleware:
if key in ("name", "value"):
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
logger.warning(msg)
- return
+ return None
continue
if isinstance(cookie[key], (bool, float, int, str)):
decoded[key] = str(cookie[key])
@@ -129,12 +158,15 @@ class CookiesMiddleware:
cookie_str += f"; {key.capitalize()}={value}"
return cookie_str
- def _get_request_cookies(self, jar, request):
+ def _get_request_cookies(
+ self, jar: CookieJar, request: Request
+ ) -> Sequence[Cookie]:
"""
Extract cookies from the Request.cookies attribute
"""
if not request.cookies:
return []
+ cookies: Iterable[Dict[str, Any]]
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index cdacc7368..8aec37cf1 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -3,19 +3,32 @@ DefaultHeaders downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
+from __future__ import annotations
+from typing import TYPE_CHECKING, Iterable, Tuple, Union
+
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
+from scrapy.http import Response
from scrapy.utils.python import without_none_values
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
class DefaultHeadersMiddleware:
- def __init__(self, headers):
- self._headers = headers
+ def __init__(self, headers: Iterable[Tuple[str, str]]):
+ self._headers: Iterable[Tuple[str, str]] = headers
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
return cls(headers.items())
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
for k, v in self._headers:
request.headers.setdefault(k, v)
+ return None
diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py
index a926ecf56..1c904c05b 100644
--- a/scrapy/downloadermiddlewares/downloadtimeout.py
+++ b/scrapy/downloadermiddlewares/downloadtimeout.py
@@ -3,23 +3,35 @@ Download timeout middleware
See documentation in docs/topics/downloader-middleware.rst
"""
+from __future__ import annotations
-from scrapy import signals
+from typing import TYPE_CHECKING, Union
+
+from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
class DownloadTimeoutMiddleware:
- def __init__(self, timeout=180):
- self._timeout = timeout
+ def __init__(self, timeout: float = 180):
+ self._timeout: float = timeout
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
- def spider_opened(self, spider):
+ def spider_opened(self, spider: Spider) -> None:
self._timeout = getattr(spider, "download_timeout", self._timeout)
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
if self._timeout:
request.meta.setdefault("download_timeout", self._timeout)
+ return None
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 5228db786..63490a37a 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -4,31 +4,44 @@ HTTP basic auth downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Union
+
from w3lib.http import basic_auth_header
-from scrapy import signals
+from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
+from scrapy.http import Response
from scrapy.utils.url import url_is_from_any_domain
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header
(http_user and http_pass spider class attributes)"""
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
o = cls()
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
- def spider_opened(self, spider):
+ def spider_opened(self, spider: Spider) -> None:
usr = getattr(spider, "http_user", "")
pwd = getattr(spider, "http_pass", "")
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
- self.domain = spider.http_auth_domain
+ self.domain = spider.http_auth_domain # type: ignore[attr-defined]
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
auth = getattr(self, "auth", None)
if auth and b"Authorization" not in request.headers:
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
request.headers[b"Authorization"] = auth
+ return None
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index a521cde7a..971473403 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,5 +1,7 @@
+from __future__ import annotations
+
from email.utils import formatdate
-from typing import Optional, Type, TypeVar
+from typing import TYPE_CHECKING, Optional, Union
from twisted.internet import defer
from twisted.internet.error import (
@@ -23,7 +25,9 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
-HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
class HttpCacheMiddleware:
@@ -49,9 +53,7 @@ class HttpCacheMiddleware:
self.stats = stats
@classmethod
- def from_crawler(
- cls: Type[HttpCacheMiddlewareTV], crawler: Crawler
- ) -> HttpCacheMiddlewareTV:
+ def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
@@ -64,7 +66,9 @@ class HttpCacheMiddleware:
def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
- def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
if request.meta.get("dont_cache", False):
return None
@@ -74,7 +78,9 @@ class HttpCacheMiddleware:
return None
# Look for cached response and check if expired
- cachedresponse = self.storage.retrieve_response(spider, request)
+ cachedresponse: Optional[Response] = self.storage.retrieve_response(
+ spider, request
+ )
if cachedresponse is None:
self.stats.inc_value("httpcache/miss", spider=spider)
if self.ignore_missing:
@@ -96,7 +102,7 @@ class HttpCacheMiddleware:
def process_response(
self, request: Request, response: Response, spider: Spider
- ) -> Response:
+ ) -> Union[Request, Response]:
if request.meta.get("dont_cache", False):
return response
@@ -111,7 +117,7 @@ class HttpCacheMiddleware:
response.headers["Date"] = formatdate(usegmt=True)
# Do not validate first-hand responses
- cachedresponse = request.meta.pop("cached_response", None)
+ cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)
if cachedresponse is None:
self.stats.inc_value("httpcache/firsthand", spider=spider)
self._cache_response(spider, response, request, cachedresponse)
@@ -127,8 +133,8 @@ class HttpCacheMiddleware:
def process_exception(
self, request: Request, exception: Exception, spider: Spider
- ) -> Optional[Response]:
- cachedresponse = request.meta.pop("cached_response", None)
+ ) -> Union[Request, Response, None]:
+ cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)
if cachedresponse is not None and isinstance(
exception, self.DOWNLOAD_EXCEPTIONS
):
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 7b1d3f829..56a58a750 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,12 +1,22 @@
+from __future__ import annotations
+
import io
import zlib
+from typing import TYPE_CHECKING, List, Optional, Union
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
+from scrapy.statscollectors import StatsCollector
from scrapy.utils.gz import gunzip
-ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
try:
import brotli
@@ -27,19 +37,24 @@ class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
- def __init__(self, stats=None):
+ def __init__(self, stats: Optional[StatsCollector] = None):
self.stats = stats
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
return cls(stats=crawler.stats)
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
+ return None
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if request.method == "HEAD":
return response
if isinstance(response, Response):
@@ -70,7 +85,7 @@ class HttpCompressionMiddleware:
return response
- def _decode(self, body, encoding):
+ def _decode(self, body: bytes, encoding: bytes) -> bytes:
if encoding == b"gzip" or encoding == b"x-gzip":
body = gunzip(body)
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index f74d84b69..335896ac1 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,16 +1,30 @@
+from __future__ import annotations
+
import base64
+from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
from urllib.parse import unquote, urlunparse
-from urllib.request import _parse_proxy, getproxies, proxy_bypass
+from urllib.request import ( # type: ignore[attr-defined]
+ _parse_proxy,
+ getproxies,
+ proxy_bypass,
+)
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
class HttpProxyMiddleware:
- def __init__(self, auth_encoding="latin-1"):
- self.auth_encoding = auth_encoding
- self.proxies = {}
+ def __init__(self, auth_encoding: Optional[str] = "latin-1"):
+ self.auth_encoding: Optional[str] = auth_encoding
+ self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {}
for type_, url in getproxies().items():
try:
self.proxies[type_] = self._get_proxy(url, type_)
@@ -20,19 +34,19 @@ class HttpProxyMiddleware:
continue
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("HTTPPROXY_ENABLED"):
raise NotConfigured
- auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
+ auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
return cls(auth_encoding)
- def _basic_auth_header(self, username, password):
+ def _basic_auth_header(self, username: str, password: str) -> bytes:
user_pass = to_bytes(
f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding
)
return base64.b64encode(user_pass)
- def _get_proxy(self, url, orig_type):
+ def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]:
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))
@@ -43,7 +57,9 @@ class HttpProxyMiddleware:
return creds, proxy_url
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
creds, proxy_url = None, None
if "proxy" in request.meta:
if request.meta["proxy"] is not None:
@@ -54,13 +70,16 @@ class HttpProxyMiddleware:
if (
# 'no_proxy' is only supported by http schemes
scheme not in ("http", "https")
- or not proxy_bypass(parsed.hostname)
+ or (parsed.hostname and not proxy_bypass(parsed.hostname))
) and scheme in self.proxies:
creds, proxy_url = self.proxies[scheme]
self._set_proxy_and_creds(request, proxy_url, creds)
+ return None
- def _set_proxy_and_creds(self, request, proxy_url, creds):
+ def _set_proxy_and_creds(
+ self, request: Request, proxy_url: Optional[str], creds: Optional[bytes]
+ ) -> None:
if proxy_url:
request.meta["proxy"] = proxy_url
elif request.meta.get("proxy") is not None:
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 65f1d2224..814b1a561 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,17 +1,29 @@
+from __future__ import annotations
+
import logging
+from typing import TYPE_CHECKING, Any, List, Union, cast
from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
-from scrapy.http import HtmlResponse
+from scrapy.http import HtmlResponse, Response
+from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
logger = logging.getLogger(__name__)
-def _build_redirect_request(source_request, *, url, **kwargs):
+def _build_redirect_request(
+ source_request: Request, *, url: str, **kwargs: Any
+) -> Request:
redirect_request = source_request.replace(
url=url,
**kwargs,
@@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs):
class BaseRedirectMiddleware:
- enabled_setting = "REDIRECT_ENABLED"
+ enabled_setting: str = "REDIRECT_ENABLED"
- def __init__(self, settings):
+ def __init__(self, settings: BaseSettings):
if not settings.getbool(self.enabled_setting):
raise NotConfigured
- self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES")
- self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST")
+ self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES")
+ self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST")
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
- def _redirect(self, redirected, request, spider, reason):
+ def _redirect(
+ self, redirected: Request, request: Request, spider: Spider, reason: Any
+ ) -> Request:
ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
redirects = request.meta.get("redirect_times", 0) + 1
@@ -67,7 +81,9 @@ class BaseRedirectMiddleware:
)
raise IgnoreRequest("max redirections reached")
- def _redirect_request_using_get(self, request, redirect_url):
+ def _redirect_request_using_get(
+ self, request: Request, redirect_url: str
+ ) -> Request:
redirect_request = _build_redirect_request(
request,
url=redirect_url,
@@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):
and meta-refresh html tag.
"""
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if (
request.meta.get("dont_redirect", False)
or response.status in getattr(spider, "handle_httpstatus_list", [])
@@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if "Location" not in response.headers or response.status not in allowed_status:
return response
+ assert response.headers["Location"] is not None
location = safe_url_string(response.headers["Location"])
if response.headers["Location"].startswith(b"//"):
request_scheme = urlparse(request.url).scheme
@@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = "METAREFRESH_ENABLED"
- def __init__(self, settings):
+ def __init__(self, settings: BaseSettings):
super().__init__(settings)
- self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS")
- self._maxdelay = settings.getint("METAREFRESH_MAXDELAY")
+ self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
+ self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if (
request.meta.get("dont_redirect", False)
or request.method == "HEAD"
@@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
return response
interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
- if url and interval < self._maxdelay:
+ if url and cast(float, interval) < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, "meta refresh")
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 380623cea..3c494de78 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,22 +9,30 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
+from __future__ import annotations
+
import warnings
from logging import Logger, getLogger
-from typing import Optional, Type, Union
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union
+from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.http import Response
from scrapy.http.request import Request
-from scrapy.settings import Settings
+from scrapy.settings import BaseSettings, Settings
from scrapy.spiders import Spider
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
retry_logger = getLogger(__name__)
-def backwards_compatibility_getattr(self, name):
+def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]:
if name == "EXCEPTIONS_TO_RETRY":
warnings.warn(
"Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
@@ -54,7 +62,7 @@ def get_retry_request(
priority_adjust: Optional[int] = None,
logger: Logger = retry_logger,
stats_base_key: str = "retry",
-):
+) -> Optional[Request]:
"""
Returns a new :class:`~scrapy.Request` object to retry the specified
request, or ``None`` if retries of the specified request have been
@@ -134,7 +142,7 @@ def get_retry_request(
class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
- def __init__(self, settings):
+ def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
@@ -153,10 +161,12 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
)
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if request.meta.get("dont_retry", False):
return response
if response.status in self.retry_http_codes:
@@ -164,13 +174,21 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
return self._retry(request, reason, spider) or response
return response
- def process_exception(self, request, exception, spider):
+ def process_exception(
+ self, request: Request, exception: Exception, spider: Spider
+ ) -> Union[Request, Response, None]:
if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
"dont_retry", False
):
return self._retry(request, exception, spider)
-
- def _retry(self, request, reason, spider):
+ return None
+
+ def _retry(
+ self,
+ request: Request,
+ reason: Union[str, Exception, Type[Exception]],
+ spider: Spider,
+ ) -> Optional[Request]:
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
return get_retry_request(
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6cab27c5a..6a0ecb7bf 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
"""
+from __future__ import annotations
+
import logging
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.python.failure import Failure
+from scrapy import Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
-from scrapy.http import Request
+from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
+from scrapy.robotstxt import RobotParser
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+
logger = logging.getLogger(__name__)
class RobotsTxtMiddleware:
- DOWNLOAD_PRIORITY = 1000
+ DOWNLOAD_PRIORITY: int = 1000
- def __init__(self, crawler):
+ def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
raise NotConfigured
- self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy")
- self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None)
- self.crawler = crawler
- self._parsers = {}
- self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER"))
+ self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
+ self._robotstxt_useragent: Optional[str] = crawler.settings.get(
+ "ROBOTSTXT_USER_AGENT", None
+ )
+ self.crawler: Crawler = crawler
+ self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}
+ self._parserimpl: RobotParser = load_object(
+ crawler.settings.get("ROBOTSTXT_PARSER")
+ )
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b"")
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
- def process_request(self, request, spider):
+ def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
if request.meta.get("dont_obey_robotstxt"):
- return
+ return None
if request.url.startswith("data:") or request.url.startswith("file:"):
- return
- d = maybeDeferred(self.robot_parser, request, spider)
+ return None
+ d: Deferred = maybeDeferred(self.robot_parser, request, spider)
d.addCallback(self.process_request_2, request, spider)
return d
- def process_request_2(self, rp, request, spider):
+ def process_request_2(
+ self, rp: Optional[RobotParser], request: Request, spider: Spider
+ ) -> None:
if rp is None:
return
- useragent = self._robotstxt_useragent
+ useragent: Union[str, bytes, None] = self._robotstxt_useragent
if not useragent:
useragent = request.headers.get(b"User-Agent", self._default_useragent)
+ assert useragent is not None
if not rp.allowed(request.url, useragent):
logger.debug(
"Forbidden by robots.txt: %(request)s",
{"request": request},
extra={"spider": spider},
)
+ assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
- def robot_parser(self, request, spider):
+ def robot_parser(
+ self, request: Request, spider: Spider
+ ) -> Union[RobotParser, Deferred, None]:
url = urlparse_cached(request)
netloc = url.netloc
@@ -75,24 +97,27 @@ class RobotsTxtMiddleware:
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
+ assert self.crawler.engine
+ assert self.crawler.stats
dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value("robotstxt/request_count")
- if isinstance(self._parsers[netloc], Deferred):
- d = Deferred()
+ parser = self._parsers[netloc]
+ if isinstance(parser, Deferred):
+ d: Deferred = Deferred()
- def cb(result):
+ def cb(result: Any) -> Any:
d.callback(result)
return result
- self._parsers[netloc].addCallback(cb)
+ parser.addCallback(cb)
return d
- return self._parsers[netloc]
+ return parser
- def _logerror(self, failure, request, spider):
+ def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure:
if failure.type is not IgnoreRequest:
logger.error(
"Error downloading %(request)s: %(f_exception)s",
@@ -102,20 +127,24 @@ class RobotsTxtMiddleware:
)
return failure
- def _parse_robots(self, response, netloc, spider):
+ def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None:
+ assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
+ assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp
rp_dfd.callback(rp)
- def _robots_error(self, failure, netloc):
+ def _robots_error(self, failure: Failure, netloc: str) -> None:
if failure.type is not IgnoreRequest:
key = f"robotstxt/exception_count/{failure.type}"
+ assert self.crawler.stats
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
+ assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = None
rp_dfd.callback(None)
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index a0f62e262..df30e8ca4 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -1,11 +1,23 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Dict, Union
+
from twisted.web import http
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+from scrapy.statscollectors import StatsCollector
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
-def get_header_size(headers):
+def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:
size = 0
for key, value in headers.items():
if isinstance(value, (list, tuple)):
@@ -14,30 +26,36 @@ def get_header_size(headers):
return size + len(b"\r\n") * (len(headers.keys()) - 1)
-def get_status_size(response_status):
+def get_status_size(response_status: int) -> int:
return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
class DownloaderStats:
- def __init__(self, stats):
- self.stats = stats
+ def __init__(self, stats: StatsCollector):
+ self.stats: StatsCollector = stats
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("DOWNLOADER_STATS"):
raise NotConfigured
+ assert crawler.stats
return cls(crawler.stats)
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
self.stats.inc_value("downloader/request_count", spider=spider)
self.stats.inc_value(
f"downloader/request_method_count/{request.method}", spider=spider
)
reqlen = len(request_httprepr(request))
self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
+ return None
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
self.stats.inc_value("downloader/response_count", spider=spider)
self.stats.inc_value(
f"downloader/response_status_count/{response.status}", spider=spider
@@ -52,9 +70,12 @@ class DownloaderStats:
self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
return response
- def process_exception(self, request, exception, spider):
+ def process_exception(
+ self, request: Request, exception: Exception, spider: Spider
+ ) -> Union[Request, Response, None]:
ex_class = global_object_name(exception.__class__)
self.stats.inc_value("downloader/exception_count", spider=spider)
self.stats.inc_value(
f"downloader/exception_type_count/{ex_class}", spider=spider
)
+ return None
diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py
index 856a275ab..92f1ec897 100644
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@@ -1,23 +1,36 @@
"""Set User-Agent header per spider or use a default value from settings"""
-from scrapy import signals
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Union
+
+from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""
- def __init__(self, user_agent="Scrapy"):
+ def __init__(self, user_agent: str = "Scrapy"):
self.user_agent = user_agent
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings["USER_AGENT"])
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
- def spider_opened(self, spider):
+ def spider_opened(self, spider: Spider) -> None:
self.user_agent = getattr(spider, "user_agent", self.user_agent)
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
if self.user_agent:
request.headers.setdefault(b"User-Agent", self.user_agent)
+ return None
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index a5329ad51..2595f328e 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,8 +1,12 @@
import re
import time
+from http.cookiejar import Cookie
from http.cookiejar import CookieJar as _CookieJar
from http.cookiejar import DefaultCookiePolicy
+from typing import Sequence
+from scrapy import Request
+from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@@ -24,7 +28,7 @@ class CookieJar:
wrsp = WrappedResponse(response)
return self.jar.extract_cookies(wrsp, wreq)
- def add_cookie_header(self, request):
+ def add_cookie_header(self, request: Request) -> None:
wreq = WrappedRequest(request)
self.policy._now = self.jar._now = int(time.time())
@@ -75,7 +79,7 @@ class CookieJar:
def set_policy(self, pol):
return self.jar.set_policy(pol)
- def make_cookies(self, response, request):
+ def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]:
wreq = WrappedRequest(request)
wrsp = WrappedResponse(response)
return self.jar.make_cookies(wrsp, wreq)
@@ -83,7 +87,7 @@ class CookieJar:
def set_cookie(self, cookie):
self.jar.set_cookie(cookie)
- def set_cookie_if_ok(self, cookie, request):
+ def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None:
self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 5c5ac4e41..6ea2bfd97 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,11 +1,20 @@
+from __future__ import annotations
+
import logging
import sys
from abc import ABCMeta, abstractmethod
+from typing import TYPE_CHECKING, Union
from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+ from scrapy.crawler import Crawler
+
logger = logging.getLogger(__name__)
@@ -31,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
- def from_crawler(cls, crawler, robotstxt_body):
+ def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
"""Parse the content of a robots.txt_ file as bytes. This must be a class method.
It must return a new instance of the parser backend.
@@ -44,14 +53,14 @@ class RobotParser(metaclass=ABCMeta):
pass
@abstractmethod
- def allowed(self, url, user_agent):
+ def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
"""Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
:param url: Absolute URL
- :type url: str
+ :type url: str or bytes
:param user_agent: User agent
- :type user_agent: str
+ :type user_agent: str or bytes
"""
pass
diff --git a/setup.cfg b/setup.cfg
index db79c5821..d23549f10 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -16,9 +16,6 @@ follow_imports = skip
# FIXME: remove the following sections once the issues are solved
-[mypy-scrapy.downloadermiddlewares.httpproxy]
-ignore_errors = True
-
[mypy-scrapy.interfaces]
ignore_errors = True