Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bump mypy, flake8, and pylint (#5738)

Andrey Rahmatullin5bd27191a2fd
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index d0fb4efd8..b9ba3335e 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -23,7 +23,7 @@ class ScrapyCommand:
exitcode = 0
- def __init__(self):
+ def __init__(self) -> None:
self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 7225e0743..69ac1cdaf 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,8 +1,22 @@
"""This module implements the Scraper component which parses responses and
extracts information from them"""
+from __future__ import annotations
+
import logging
from collections import deque
-from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union
+from typing import (
+ Any,
+ AsyncGenerator,
+ AsyncIterable,
+ Deque,
+ Generator,
+ Iterable,
+ Optional,
+ Set,
+ TYPE_CHECKING,
+ Tuple,
+ Union,
+)
from itemadapter import is_item
from twisted.internet.defer import Deferred, inlineCallbacks
@@ -26,6 +40,10 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+
+
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
@@ -75,7 +93,7 @@ class Slot:
class Scraper:
- def __init__(self, crawler):
+ def __init__(self, crawler: Crawler) -> None:
self.slot: Optional[Slot] = None
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
@@ -174,6 +192,7 @@ class Scraper:
def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
exc = _failure.value
if isinstance(exc, CloseSpider):
+ assert self.crawler.engine is not None # typing
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
return
logkws = self.logformatter.spider_error(_failure, request, response, spider)
@@ -214,6 +233,7 @@ class Scraper:
"""
assert self.slot is not None # typing
if isinstance(output, Request):
+ assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
elif is_item(output):
self.slot.itemproc_size += 1
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 04c6891e3..4700a30ab 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,7 +4,7 @@ import logging
import pprint
import signal
import warnings
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@@ -109,7 +109,7 @@ class Crawler:
self.settings.freeze()
self.crawling = False
self.spider = None
- self.engine = None
+ self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index fb2d0f165..7626946ec 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -140,8 +140,7 @@ class Response(object_ref):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, cb_kwargs=None, flags=None):
- # type: (...) -> Request
+ dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
@@ -179,8 +178,8 @@ class Response(object_ref):
def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, cb_kwargs=None, flags=None):
- # type: (...) -> Generator[Request, None, None]
+ dont_filter=False, errback=None, cb_kwargs=None,
+ flags=None) -> Generator[Request, None, None]:
"""
.. versionadded:: 2.0
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index bfcde878d..da81d0a4a 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -142,8 +142,7 @@ class TextResponse(Response):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
- dont_filter=False, errback=None, cb_kwargs=None, flags=None):
- # type: (...) -> Request
+ dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
@@ -184,8 +183,7 @@ class TextResponse(Response):
def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
dont_filter=False, errback=None, cb_kwargs=None, flags=None,
- css=None, xpath=None):
- # type: (...) -> Generator[Request, None, None]
+ css=None, xpath=None) -> Generator[Request, None, None]:
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 8d7e5a602..431bd76dc 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
import logging
import pprint
from collections import defaultdict, deque
-from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast
+from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast
from twisted.internet.defer import Deferred
@@ -19,7 +19,7 @@ class MiddlewareManager:
component_name = 'foo middleware'
- def __init__(self, *middlewares):
+ def __init__(self, *middlewares: Any) -> None:
self.middlewares = middlewares
# Only process_spider_output and process_spider_exception can be None.
# Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
diff --git a/tox.ini b/tox.ini
index eee99cb2d..4d0f0291b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -38,7 +38,7 @@ install_command =
basepython = python3
deps =
lxml-stubs==0.2.0
- mypy==0.982
+ mypy==0.991
types-attrs==19.1.0
types-pyOpenSSL==21.0.0
types-setuptools==57.0.0
@@ -58,7 +58,7 @@ deps =
{[testenv]deps}
# Twisted[http2] is required to import some files
Twisted[http2]>=17.9.0
- flake8==5.0.4
+ flake8==6.0.0
commands =
flake8 {posargs:docs scrapy tests}
@@ -67,7 +67,7 @@ commands =
basepython = python3.8
deps =
{[testenv:extra-deps]deps}
- pylint==2.15.3
+ pylint==2.15.6
commands =
pylint conftest.py docs extras scrapy setup.py tests