Scrapy, a fast high-level web crawling & scraping framework for Python.
Some type hints
Eugenio Lacuestaef09e0d10fc9
diff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex 4326ca4aa..8a8065bf2 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -22,7 +22,7 @@ __all__ = [# Scrapy and Twisted versions-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()+__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))twisted_version = (_txv.major, _txv.minor, _txv.micro)diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 23ccffcd9..6e77551c6 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -3,6 +3,8 @@ Base class for Scrapy commands"""import osfrom optparse import OptionGroup+from typing import Any, Dict+from twisted.python import failurefrom scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli@@ -15,7 +17,7 @@ class ScrapyCommand:crawler_process = None# default settings to be used for this command instead of global defaults- default_settings = {}+ default_settings: Dict[str, Any] = {}exitcode = 0diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 83ee074da..52118db1b 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -1,5 +1,6 @@import jsonimport logging+from typing import Dictfrom itemadapter import is_item, ItemAdapterfrom w3lib.url import is_url@@ -10,6 +11,7 @@ from scrapy.utils import displayfrom scrapy.utils.spider import iterate_spider_output, spidercls_for_requestfrom scrapy.exceptions import UsageError+logger = logging.getLogger(__name__)@@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand):requires_project = Truespider = None- items = {}- requests = {}+ items: Dict[int, list] = {}+ requests: Dict[int, list] = {}first_response = Nonediff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex db0a56e56..b47e55092 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -1,16 +1,77 @@-import sysimport re+import sysfrom functools import wrapsfrom inspect import getmembers+from typing import Dictfrom unittest import TestCasefrom scrapy.http import Request-from scrapy.utils.spider import iterate_spider_outputfrom scrapy.utils.python import get_spec+from scrapy.utils.spider import iterate_spider_output+++class Contract:+ """ Abstract class for contracts """+ request_cls = None++ def __init__(self, method, *args):+ self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')+ self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')+ self.args = args++ def add_pre_hook(self, request, results):+ if hasattr(self, 'pre_process'):+ cb = request.callback++ @wraps(cb)+ def wrapper(response, **cb_kwargs):+ try:+ results.startTest(self.testcase_pre)+ self.pre_process(response)+ results.stopTest(self.testcase_pre)+ except AssertionError:+ results.addFailure(self.testcase_pre, sys.exc_info())+ except Exception:+ results.addError(self.testcase_pre, sys.exc_info())+ else:+ results.addSuccess(self.testcase_pre)+ finally:+ return list(iterate_spider_output(cb(response, **cb_kwargs)))++ request.callback = wrapper++ return request++ def add_post_hook(self, request, results):+ if hasattr(self, 'post_process'):+ cb = request.callback++ @wraps(cb)+ def wrapper(response, **cb_kwargs):+ output = list(iterate_spider_output(cb(response, **cb_kwargs)))+ try:+ results.startTest(self.testcase_post)+ self.post_process(output)+ results.stopTest(self.testcase_post)+ except AssertionError:+ results.addFailure(self.testcase_post, sys.exc_info())+ except Exception:+ results.addError(self.testcase_post, sys.exc_info())+ else:+ results.addSuccess(self.testcase_post)+ finally:+ return output++ request.callback = wrapper++ return request++ def adjust_request_args(self, args):+ return argsclass ContractsManager:- contracts = {}+ contracts: Dict[str, Contract] = {}def __init__(self, contracts):for contract in contracts:@@ -107,66 +168,6 @@ class ContractsManager:request.errback = eb_wrapper-class Contract:- """ Abstract class for contracts """- request_cls = None-- def __init__(self, method, *args):- self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')- self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')- self.args = args-- def add_pre_hook(self, request, results):- if hasattr(self, 'pre_process'):- cb = request.callback-- @wraps(cb)- def wrapper(response, **cb_kwargs):- try:- results.startTest(self.testcase_pre)- self.pre_process(response)- results.stopTest(self.testcase_pre)- except AssertionError:- results.addFailure(self.testcase_pre, sys.exc_info())- except Exception:- results.addError(self.testcase_pre, sys.exc_info())- else:- results.addSuccess(self.testcase_pre)- finally:- return list(iterate_spider_output(cb(response, **cb_kwargs)))-- request.callback = wrapper-- return request-- def add_post_hook(self, request, results):- if hasattr(self, 'post_process'):- cb = request.callback-- @wraps(cb)- def wrapper(response, **cb_kwargs):- output = list(iterate_spider_output(cb(response, **cb_kwargs)))- try:- results.startTest(self.testcase_post)- self.post_process(output)- results.stopTest(self.testcase_post)- except AssertionError:- results.addFailure(self.testcase_post, sys.exc_info())- except Exception:- results.addError(self.testcase_post, sys.exc_info())- else:- results.addSuccess(self.testcase_post)- finally:- return output-- request.callback = wrapper-- return request-- def adjust_request_args(self, args):- return args--def _create_testcase(method, desc):spider = method.__self__.namediff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex 0c97e6999..bf4ae7b45 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -1,10 +1,16 @@+import reimport time-from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicyfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_unicode+# Defined in the http.cookiejar module, but undocumented:+# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527+IPV4_RE = re.compile(r"\.\d+$", re.ASCII)++class CookieJar:def __init__(self, policy=None, check_expired_frequency=10000):self.policy = policy or DefaultCookiePolicy()diff --git a/scrapy/item.py b/scrapy/item.pyindex af3849302..2ccd7ad18 100644--- a/scrapy/item.py+++ b/scrapy/item.py@@ -8,6 +8,7 @@ from abc import ABCMetafrom collections.abc import MutableMappingfrom copy import deepcopyfrom pprint import pformat+from typing import Dictfrom warnings import warnfrom scrapy.utils.deprecate import ScrapyDeprecationWarning@@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta):class DictItem(MutableMapping, BaseItem):- fields = {}+ fields: Dict[str, Field] = {}def __new__(cls, *args, **kwargs):if issubclass(cls, DictItem) and not issubclass(cls, Item):diff --git a/scrapy/mail.py b/scrapy/mail.pyindex 7d7a2c435..2a25ccd44 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -9,7 +9,7 @@ from email.mime.base import MIMEBasefrom email.mime.multipart import MIMEMultipartfrom email.mime.nonmultipart import MIMENonMultipartfrom email.mime.text import MIMEText-from email.utils import COMMASPACE, formatdate+from email.utils import formatdatefrom io import BytesIOfrom twisted.internet import defer, ssl@@ -21,6 +21,11 @@ from scrapy.utils.python import to_byteslogger = logging.getLogger(__name__)+# Defined in the email.utils module, but undocumented:+# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42+COMMASPACE = ", "++def _to_bytes_or_none(text):if text is None:return Nonediff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex f81041376..608c0eea5 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response whichoriginated it."""import warnings+from typing import Tuplefrom urllib.parse import urlparsefrom w3lib.url import safe_url_string-from scrapy.http import Request, Response-from scrapy.exceptions import NotConfiguredfrom scrapy import signals-from scrapy.utils.python import to_unicode+from scrapy.exceptions import NotConfigured+from scrapy.http import Request, Responsefrom scrapy.utils.misc import load_object+from scrapy.utils.python import to_unicodefrom scrapy.utils.url import strip_url@@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"class ReferrerPolicy:- NOREFERRER_SCHEMES = LOCAL_SCHEMES+ NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES+ name: strdef referrer(self, response_url, request_url):raise NotImplementedError()@@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy):is to be sent along with requests made from a particular request client to any origin.The header will be omitted entirely."""- name = POLICY_NO_REFERRER+ name: str = POLICY_NO_REFERRERdef referrer(self, response_url, request_url):return None@@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):This is a user agent's default behavior, if no policy is otherwise specified."""- name = POLICY_NO_REFERRER_WHEN_DOWNGRADE+ name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADEdef referrer(self, response_url, request_url):if not self.tls_protected(response_url) or self.tls_protected(request_url):@@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy):Cross-origin requests, on the other hand, will contain no referrer information.A Referer HTTP header will not be sent."""- name = POLICY_SAME_ORIGIN+ name: str = POLICY_SAME_ORIGINdef referrer(self, response_url, request_url):if self.origin(response_url) == self.origin(request_url):@@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy):when making both same-origin requests and cross-origin requestsfrom a particular request client."""- name = POLICY_ORIGIN+ name: str = POLICY_ORIGINdef referrer(self, response_url, request_url):return self.origin_referrer(response_url)@@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy):on the other hand, will contain no referrer information.A Referer HTTP header will not be sent."""- name = POLICY_STRICT_ORIGIN+ name: str = POLICY_STRICT_ORIGINdef referrer(self, response_url, request_url):if (@@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):is sent as referrer information when making cross-origin requestsfrom a particular request client."""- name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN+ name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGINdef referrer(self, response_url, request_url):origin = self.origin(response_url)@@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):on the other hand, will contain no referrer information.A Referer HTTP header will not be sent."""- name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN+ name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGINdef referrer(self, response_url, request_url):origin = self.origin(response_url)@@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):to insecure origins.Carefully consider the impact of setting such a policy for potentially sensitive documents."""- name = POLICY_UNSAFE_URL+ name: str = POLICY_UNSAFE_URLdef referrer(self, response_url, request_url):return self.stripped_referrer(response_url)@@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):with the addition that "Referer" is not sent if the parent request wasusing ``file://`` or ``s3://`` scheme."""- NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')- name = POLICY_SCRAPY_DEFAULT+ NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3')+ name: str = POLICY_SCRAPY_DEFAULT_policy_classes = {p.name: p for p in (diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.pyindex c8d4391b1..a90f1d278 100644--- a/scrapy/utils/httpobj.py+++ b/scrapy/utils/httpobj.py@@ -1,13 +1,16 @@"""Helper functions for scrapy.http objects (Request, Response)"""-import weakref-from urllib.parse import urlparse+from typing import Union+from urllib.parse import urlparse, ParseResult+from weakref import WeakKeyDictionary+from scrapy.http import Request, Response-_urlparse_cache = weakref.WeakKeyDictionary()+_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()-def urlparse_cached(request_or_response):++def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:"""Return urlparse.urlparse caching the result, where the argument can be aRequest or Response object"""diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 12c03d78e..66736b42f 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -4,20 +4,27 @@ scrapy.http.Request objects"""import hashlib-import weakref+from typing import Dict, Iterable, Optional, Tuple, Unionfrom urllib.parse import urlunparse+from weakref import WeakKeyDictionaryfrom w3lib.http import basic_auth_headerfrom w3lib.url import canonicalize_url+from scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode-_fingerprint_cache = weakref.WeakKeyDictionary()+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"+_fingerprint_cache = WeakKeyDictionary()-def request_fingerprint(request, include_headers=None, keep_fragments=False):+def request_fingerprint(+ request: Request,+ include_headers: Optional[Iterable[Union[bytes, str]]] = None,+ keep_fragments: bool = False,+):"""Return the request fingerprint.@@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):(for instance when handling requests with a headless browser)."""+ headers: Optional[Tuple[bytes, ...]] = Noneif include_headers:- include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))+ headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))cache = _fingerprint_cache.setdefault(request, {})- cache_key = (include_headers, keep_fragments)+ cache_key = (headers, keep_fragments)if cache_key not in cache:fp = hashlib.sha1()fp.update(to_bytes(request.method))fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))fp.update(request.body or b'')- if include_headers:- for hdr in include_headers:+ if headers:+ for hdr in headers:if hdr in request.headers:fp.update(hdr)for v in request.headers.getlist(hdr):@@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):return cache[cache_key]-def request_authenticate(request, username, password):+def request_authenticate(request: Request, username: str, password: str) -> None:"""Autenticate the given request (in place) using the HTTP basic accessauthentication mechanism (RFC 2617) and the given username and password"""request.headers['Authorization'] = basic_auth_header(username, password)-def request_httprepr(request):+def request_httprepr(request: Request) -> bytes:"""Return the raw HTTP representation (as bytes) of the given request.This is provided only for reference since it's not the actual stream ofbytes that will be send when performing the request (that's controlled@@ -92,7 +100,7 @@ def request_httprepr(request):return s-def referer_str(request):+def referer_str(request: Request) -> Optional[str]:""" Return Referer HTTP header suitable for logging. """referrer = request.headers.get('Referer')if referrer is None:diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex 99b089b6f..b3ef7b463 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -3,19 +3,23 @@ This module provides some useful functions for working withscrapy.http.Response objects"""import os-import weakrefimport webbrowserimport tempfile+from typing import Any, Callable, Iterable, Optional, Tuple, Union+from weakref import WeakKeyDictionary++import scrapy+from scrapy.http.response import Responsefrom twisted.web import httpfrom scrapy.utils.python import to_bytes, to_unicodefrom w3lib import html-_baseurl_cache = weakref.WeakKeyDictionary()+_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()-def get_base_url(response):+def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:"""Return the base url of the given response, joined with the response url"""if response not in _baseurl_cache:text = response.text[0:4096]@@ -23,10 +27,13 @@ def get_base_url(response):return _baseurl_cache[response]-_metaref_cache = weakref.WeakKeyDictionary()+_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()-def get_meta_refresh(response, ignore_tags=('script', 'noscript')):+def get_meta_refresh(+ response: "scrapy.http.response.text.TextResponse",+ ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),+) -> Union[Tuple[None, None], Tuple[float, str]]:"""Parse the http-equiv refrsh parameter from the given response"""if response not in _metaref_cache:text = response.text[0:4096]@@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):return _metaref_cache[response]-def response_status_message(status):+def response_status_message(status: Union[bytes, float, int, str]) -> str:"""Return status code plus status text descriptive message"""- message = http.RESPONSES.get(int(status), "Unknown Status")- return f'{status} {to_unicode(message)}'+ status_int = int(status)+ message = http.RESPONSES.get(status_int, "Unknown Status")+ return f'{status_int} {to_unicode(message)}'-def response_httprepr(response):+def response_httprepr(response: Response) -> bytes:"""Return raw HTTP representation (as bytes) of the given response. Thisis provided only for reference, since it's not the exact stream of bytesthat was received (that's not exposed by Twisted).@@ -60,7 +68,10 @@ def response_httprepr(response):return b"".join(values)-def open_in_browser(response, _openfunc=webbrowser.open):+def open_in_browser(+ response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"],+ _openfunc: Callable[[str], Any] = webbrowser.open,+) -> Any:"""Open the given response in a local web browser, populating the <base>tag for external links to work"""diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.pyindex 3e40acd69..b0c6a2424 100644--- a/scrapy/utils/trackref.py+++ b/scrapy/utils/trackref.py@@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just analias to object in that case)."""-import weakref-from time import time-from operator import itemgetterfrom collections import defaultdict+from operator import itemgetter+from time import time+from typing import DefaultDict+from weakref import WeakKeyDictionaryNoneType = type(None)-live_refs = defaultdict(weakref.WeakKeyDictionary)+live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)class object_ref:diff --git a/setup.cfg b/setup.cfgindex 0c9f6b963..89b4ec57e 100644--- a/setup.cfg+++ b/setup.cfg@@ -10,54 +10,18 @@ follow_imports = skip# FIXME: remove the following sections once the issues are solved-[mypy-scrapy]-ignore_errors = True--[mypy-scrapy.commands]-ignore_errors = True--[mypy-scrapy.commands.parse]-ignore_errors = True-[mypy-scrapy.downloadermiddlewares.httpproxy]ignore_errors = True-[mypy-scrapy.contracts]-ignore_errors = True-[mypy-scrapy.interfaces]ignore_errors = True-[mypy-scrapy.item]-ignore_errors = True--[mypy-scrapy.http.cookies]-ignore_errors = True--[mypy-scrapy.mail]-ignore_errors = True-[mypy-scrapy.pipelines.images]ignore_errors = True[mypy-scrapy.settings.default_settings]ignore_errors = True-[mypy-scrapy.spidermiddlewares.referer]-ignore_errors = True--[mypy-scrapy.utils.httpobj]-ignore_errors = True--[mypy-scrapy.utils.request]-ignore_errors = True--[mypy-scrapy.utils.response]-ignore_errors = True--[mypy-scrapy.utils.trackref]-ignore_errors = True-[mypy-tests.mocks.dummydbm]ignore_errors = True