Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Some type hints

Eugenio Lacuestaef09e0d10fc9
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 4326ca4aa..8a8065bf2 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -22,7 +22,7 @@ __all__ = [
# Scrapy and Twisted versions
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 23ccffcd9..6e77551c6 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -3,6 +3,8 @@ Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
+from typing import Any, Dict
+
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@@ -15,7 +17,7 @@ class ScrapyCommand:
crawler_process = None
# default settings to be used for this command instead of global defaults
- default_settings = {}
+ default_settings: Dict[str, Any] = {}
exitcode = 0
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 83ee074da..52118db1b 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,5 +1,6 @@
import json
import logging
+from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
@@ -10,6 +11,7 @@ from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
+
logger = logging.getLogger(__name__)
@@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
- items = {}
- requests = {}
+ items: Dict[int, list] = {}
+ requests: Dict[int, list] = {}
first_response = None
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index db0a56e56..b47e55092 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,16 +1,77 @@
-import sys
import re
+import sys
from functools import wraps
from inspect import getmembers
+from typing import Dict
from unittest import TestCase
from scrapy.http import Request
-from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
+from scrapy.utils.spider import iterate_spider_output
+
+
+class Contract:
+ """ Abstract class for contracts """
+ request_cls = None
+
+ def __init__(self, method, *args):
+ self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
+ self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
+ self.args = args
+
+ def add_pre_hook(self, request, results):
+ if hasattr(self, 'pre_process'):
+ cb = request.callback
+
+ @wraps(cb)
+ def wrapper(response, **cb_kwargs):
+ try:
+ results.startTest(self.testcase_pre)
+ self.pre_process(response)
+ results.stopTest(self.testcase_pre)
+ except AssertionError:
+ results.addFailure(self.testcase_pre, sys.exc_info())
+ except Exception:
+ results.addError(self.testcase_pre, sys.exc_info())
+ else:
+ results.addSuccess(self.testcase_pre)
+ finally:
+ return list(iterate_spider_output(cb(response, **cb_kwargs)))
+
+ request.callback = wrapper
+
+ return request
+
+ def add_post_hook(self, request, results):
+ if hasattr(self, 'post_process'):
+ cb = request.callback
+
+ @wraps(cb)
+ def wrapper(response, **cb_kwargs):
+ output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+ try:
+ results.startTest(self.testcase_post)
+ self.post_process(output)
+ results.stopTest(self.testcase_post)
+ except AssertionError:
+ results.addFailure(self.testcase_post, sys.exc_info())
+ except Exception:
+ results.addError(self.testcase_post, sys.exc_info())
+ else:
+ results.addSuccess(self.testcase_post)
+ finally:
+ return output
+
+ request.callback = wrapper
+
+ return request
+
+ def adjust_request_args(self, args):
+ return args
class ContractsManager:
- contracts = {}
+ contracts: Dict[str, Contract] = {}
def __init__(self, contracts):
for contract in contracts:
@@ -107,66 +168,6 @@ class ContractsManager:
request.errback = eb_wrapper
-class Contract:
- """ Abstract class for contracts """
- request_cls = None
-
- def __init__(self, method, *args):
- self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
- self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
- self.args = args
-
- def add_pre_hook(self, request, results):
- if hasattr(self, 'pre_process'):
- cb = request.callback
-
- @wraps(cb)
- def wrapper(response, **cb_kwargs):
- try:
- results.startTest(self.testcase_pre)
- self.pre_process(response)
- results.stopTest(self.testcase_pre)
- except AssertionError:
- results.addFailure(self.testcase_pre, sys.exc_info())
- except Exception:
- results.addError(self.testcase_pre, sys.exc_info())
- else:
- results.addSuccess(self.testcase_pre)
- finally:
- return list(iterate_spider_output(cb(response, **cb_kwargs)))
-
- request.callback = wrapper
-
- return request
-
- def add_post_hook(self, request, results):
- if hasattr(self, 'post_process'):
- cb = request.callback
-
- @wraps(cb)
- def wrapper(response, **cb_kwargs):
- output = list(iterate_spider_output(cb(response, **cb_kwargs)))
- try:
- results.startTest(self.testcase_post)
- self.post_process(output)
- results.stopTest(self.testcase_post)
- except AssertionError:
- results.addFailure(self.testcase_post, sys.exc_info())
- except Exception:
- results.addError(self.testcase_post, sys.exc_info())
- else:
- results.addSuccess(self.testcase_post)
- finally:
- return output
-
- request.callback = wrapper
-
- return request
-
- def adjust_request_args(self, args):
- return args
-
-
def _create_testcase(method, desc):
spider = method.__self__.name
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 0c97e6999..bf4ae7b45 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,10 +1,16 @@
+import re
import time
-from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
+# Defined in the http.cookiejar module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
+IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
+
+
class CookieJar:
def __init__(self, policy=None, check_expired_frequency=10000):
self.policy = policy or DefaultCookiePolicy()
diff --git a/scrapy/item.py b/scrapy/item.py
index af3849302..2ccd7ad18 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -8,6 +8,7 @@ from abc import ABCMeta
from collections.abc import MutableMapping
from copy import deepcopy
from pprint import pformat
+from typing import Dict
from warnings import warn
from scrapy.utils.deprecate import ScrapyDeprecationWarning
@@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta):
class DictItem(MutableMapping, BaseItem):
- fields = {}
+ fields: Dict[str, Field] = {}
def __new__(cls, *args, **kwargs):
if issubclass(cls, DictItem) and not issubclass(cls, Item):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 7d7a2c435..2a25ccd44 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -9,7 +9,7 @@ from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.nonmultipart import MIMENonMultipart
from email.mime.text import MIMEText
-from email.utils import COMMASPACE, formatdate
+from email.utils import formatdate
from io import BytesIO
from twisted.internet import defer, ssl
@@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes
logger = logging.getLogger(__name__)
+# Defined in the email.utils module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42
+COMMASPACE = ", "
+
+
def _to_bytes_or_none(text):
if text is None:
return None
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..608c0eea5 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
import warnings
+from typing import Tuple
from urllib.parse import urlparse
from w3lib.url import safe_url_string
-from scrapy.http import Request, Response
-from scrapy.exceptions import NotConfigured
from scrapy import signals
-from scrapy.utils.python import to_unicode
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Request, Response
from scrapy.utils.misc import load_object
+from scrapy.utils.python import to_unicode
from scrapy.utils.url import strip_url
@@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy:
- NOREFERRER_SCHEMES = LOCAL_SCHEMES
+ NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
+ name: str
def referrer(self, response_url, request_url):
raise NotImplementedError()
@@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy):
is to be sent along with requests made from a particular request client to any origin.
The header will be omitted entirely.
"""
- name = POLICY_NO_REFERRER
+ name: str = POLICY_NO_REFERRER
def referrer(self, response_url, request_url):
return None
@@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
This is a user agent's default behavior, if no policy is otherwise specified.
"""
- name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
+ name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
def referrer(self, response_url, request_url):
if not self.tls_protected(response_url) or self.tls_protected(request_url):
@@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy):
Cross-origin requests, on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
- name = POLICY_SAME_ORIGIN
+ name: str = POLICY_SAME_ORIGIN
def referrer(self, response_url, request_url):
if self.origin(response_url) == self.origin(request_url):
@@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy):
when making both same-origin requests and cross-origin requests
from a particular request client.
"""
- name = POLICY_ORIGIN
+ name: str = POLICY_ORIGIN
def referrer(self, response_url, request_url):
return self.origin_referrer(response_url)
@@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy):
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
- name = POLICY_STRICT_ORIGIN
+ name: str = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
if (
@@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
is sent as referrer information when making cross-origin requests
from a particular request client.
"""
- name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
+ name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
origin = self.origin(response_url)
@@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
- name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+ name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
origin = self.origin(response_url)
@@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
to insecure origins.
Carefully consider the impact of setting such a policy for potentially sensitive documents.
"""
- name = POLICY_UNSAFE_URL
+ name: str = POLICY_UNSAFE_URL
def referrer(self, response_url, request_url):
return self.stripped_referrer(response_url)
@@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
with the addition that "Referer" is not sent if the parent request was
using ``file://`` or ``s3://`` scheme.
"""
- NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
- name = POLICY_SCRAPY_DEFAULT
+ NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3')
+ name: str = POLICY_SCRAPY_DEFAULT
_policy_classes = {p.name: p for p in (
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index c8d4391b1..a90f1d278 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,13 +1,16 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
-import weakref
-from urllib.parse import urlparse
+from typing import Union
+from urllib.parse import urlparse, ParseResult
+from weakref import WeakKeyDictionary
+from scrapy.http import Request, Response
-_urlparse_cache = weakref.WeakKeyDictionary()
+_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
-def urlparse_cached(request_or_response):
+
+def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
"""Return urlparse.urlparse caching the result, where the argument can be a
Request or Response object
"""
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 12c03d78e..66736b42f 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,20 +4,27 @@ scrapy.http.Request objects
"""
import hashlib
-import weakref
+from typing import Dict, Iterable, Optional, Tuple, Union
from urllib.parse import urlunparse
+from weakref import WeakKeyDictionary
from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
+from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
-_fingerprint_cache = weakref.WeakKeyDictionary()
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_fingerprint_cache = WeakKeyDictionary()
-def request_fingerprint(request, include_headers=None, keep_fragments=False):
+def request_fingerprint(
+ request: Request,
+ include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+ keep_fragments: bool = False,
+):
"""
Return the request fingerprint.
@@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
(for instance when handling requests with a headless browser).
"""
+ headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
- include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+ headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
cache = _fingerprint_cache.setdefault(request, {})
- cache_key = (include_headers, keep_fragments)
+ cache_key = (headers, keep_fragments)
if cache_key not in cache:
fp = hashlib.sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
fp.update(request.body or b'')
- if include_headers:
- for hdr in include_headers:
+ if headers:
+ for hdr in headers:
if hdr in request.headers:
fp.update(hdr)
for v in request.headers.getlist(hdr):
@@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
return cache[cache_key]
-def request_authenticate(request, username, password):
+def request_authenticate(request: Request, username: str, password: str) -> None:
"""Autenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
request.headers['Authorization'] = basic_auth_header(username, password)
-def request_httprepr(request):
+def request_httprepr(request: Request) -> bytes:
"""Return the raw HTTP representation (as bytes) of the given request.
This is provided only for reference since it's not the actual stream of
bytes that will be send when performing the request (that's controlled
@@ -92,7 +100,7 @@ def request_httprepr(request):
return s
-def referer_str(request):
+def referer_str(request: Request) -> Optional[str]:
""" Return Referer HTTP header suitable for logging. """
referrer = request.headers.get('Referer')
if referrer is None:
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 99b089b6f..b3ef7b463 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,19 +3,23 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
import os
-import weakref
import webbrowser
import tempfile
+from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import scrapy
+from scrapy.http.response import Response
from twisted.web import http
from scrapy.utils.python import to_bytes, to_unicode
from w3lib import html
-_baseurl_cache = weakref.WeakKeyDictionary()
+_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
-def get_base_url(response):
+def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
text = response.text[0:4096]
@@ -23,10 +27,13 @@ def get_base_url(response):
return _baseurl_cache[response]
-_metaref_cache = weakref.WeakKeyDictionary()
+_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()
-def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
+def get_meta_refresh(
+ response: "scrapy.http.response.text.TextResponse",
+ ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
+) -> Union[Tuple[None, None], Tuple[float, str]]:
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
@@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
return _metaref_cache[response]
-def response_status_message(status):
+def response_status_message(status: Union[bytes, float, int, str]) -> str:
"""Return status code plus status text descriptive message
"""
- message = http.RESPONSES.get(int(status), "Unknown Status")
- return f'{status} {to_unicode(message)}'
+ status_int = int(status)
+ message = http.RESPONSES.get(status_int, "Unknown Status")
+ return f'{status_int} {to_unicode(message)}'
-def response_httprepr(response):
+def response_httprepr(response: Response) -> bytes:
"""Return raw HTTP representation (as bytes) of the given response. This
is provided only for reference, since it's not the exact stream of bytes
that was received (that's not exposed by Twisted).
@@ -60,7 +68,10 @@ def response_httprepr(response):
return b"".join(values)
-def open_in_browser(response, _openfunc=webbrowser.open):
+def open_in_browser(
+ response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"],
+ _openfunc: Callable[[str], Any] = webbrowser.open,
+) -> Any:
"""Open the given response in a local web browser, populating the <base>
tag for external links to work
"""
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 3e40acd69..b0c6a2424 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an
alias to object in that case).
"""
-import weakref
-from time import time
-from operator import itemgetter
from collections import defaultdict
+from operator import itemgetter
+from time import time
+from typing import DefaultDict
+from weakref import WeakKeyDictionary
NoneType = type(None)
-live_refs = defaultdict(weakref.WeakKeyDictionary)
+live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
class object_ref:
diff --git a/setup.cfg b/setup.cfg
index 0c9f6b963..89b4ec57e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -10,54 +10,18 @@ follow_imports = skip
# FIXME: remove the following sections once the issues are solved
-[mypy-scrapy]
-ignore_errors = True
-
-[mypy-scrapy.commands]
-ignore_errors = True
-
-[mypy-scrapy.commands.parse]
-ignore_errors = True
-
[mypy-scrapy.downloadermiddlewares.httpproxy]
ignore_errors = True
-[mypy-scrapy.contracts]
-ignore_errors = True
-
[mypy-scrapy.interfaces]
ignore_errors = True
-[mypy-scrapy.item]
-ignore_errors = True
-
-[mypy-scrapy.http.cookies]
-ignore_errors = True
-
-[mypy-scrapy.mail]
-ignore_errors = True
-
[mypy-scrapy.pipelines.images]
ignore_errors = True
[mypy-scrapy.settings.default_settings]
ignore_errors = True
-[mypy-scrapy.spidermiddlewares.referer]
-ignore_errors = True
-
-[mypy-scrapy.utils.httpobj]
-ignore_errors = True
-
-[mypy-scrapy.utils.request]
-ignore_errors = True
-
-[mypy-scrapy.utils.response]
-ignore_errors = True
-
-[mypy-scrapy.utils.trackref]
-ignore_errors = True
-
[mypy-tests.mocks.dummydbm]
ignore_errors = True