Scrapy, a fast high-level web crawling & scraping framework for Python.
Bump mypy, flake8, and pylint (#5738)
Andrey Rahmatullin5bd27191a2fd
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex d0fb4efd8..b9ba3335e 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -23,7 +23,7 @@ class ScrapyCommand:exitcode = 0- def __init__(self):+ def __init__(self) -> None:self.settings: Any = None # set in scrapy.cmdlinedef set_crawler(self, crawler):diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 7225e0743..69ac1cdaf 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -1,8 +1,22 @@"""This module implements the Scraper component which parses responses andextracts information from them"""+from __future__ import annotations+import loggingfrom collections import deque-from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union+from typing import (+ Any,+ AsyncGenerator,+ AsyncIterable,+ Deque,+ Generator,+ Iterable,+ Optional,+ Set,+ TYPE_CHECKING,+ Tuple,+ Union,+)from itemadapter import is_itemfrom twisted.internet.defer import Deferred, inlineCallbacks@@ -26,6 +40,10 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_valuefrom scrapy.utils.spider import iterate_spider_output+if TYPE_CHECKING:+ from scrapy.crawler import Crawler++QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]@@ -75,7 +93,7 @@ class Slot:class Scraper:- def __init__(self, crawler):+ def __init__(self, crawler: Crawler) -> None:self.slot: Optional[Slot] = Noneself.spidermw = SpiderMiddlewareManager.from_crawler(crawler)itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])@@ -174,6 +192,7 @@ class Scraper:def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:exc = _failure.valueif isinstance(exc, CloseSpider):+ assert self.crawler.engine is not None # typingself.crawler.engine.close_spider(spider, exc.reason or 'cancelled')returnlogkws = self.logformatter.spider_error(_failure, request, response, spider)@@ -214,6 +233,7 @@ class Scraper:"""assert self.slot is not None # typingif isinstance(output, Request):+ assert self.crawler.engine is not None # typingself.crawler.engine.crawl(request=output)elif is_item(output):self.slot.itemproc_size += 1diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 04c6891e3..4700a30ab 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -4,7 +4,7 @@ import loggingimport pprintimport signalimport warnings-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Optionalfrom twisted.internet import deferfrom zope.interface.exceptions import DoesNotImplement@@ -109,7 +109,7 @@ class Crawler:self.settings.freeze()self.crawling = Falseself.spider = None- self.engine = None+ self.engine: Optional[ExecutionEngine] = None@defer.inlineCallbacksdef crawl(self, *args, **kwargs):diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex fb2d0f165..7626946ec 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -140,8 +140,7 @@ class Response(object_ref):def follow(self, url, callback=None, method='GET', headers=None, body=None,cookies=None, meta=None, encoding='utf-8', priority=0,- dont_filter=False, errback=None, cb_kwargs=None, flags=None):- # type: (...) -> Request+ dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:"""Return a :class:`~.Request` instance to follow a link ``url``.It accepts the same arguments as ``Request.__init__`` method,@@ -179,8 +178,8 @@ class Response(object_ref):def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,cookies=None, meta=None, encoding='utf-8', priority=0,- dont_filter=False, errback=None, cb_kwargs=None, flags=None):- # type: (...) -> Generator[Request, None, None]+ dont_filter=False, errback=None, cb_kwargs=None,+ flags=None) -> Generator[Request, None, None]:""".. versionadded:: 2.0diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex bfcde878d..da81d0a4a 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -142,8 +142,7 @@ class TextResponse(Response):def follow(self, url, callback=None, method='GET', headers=None, body=None,cookies=None, meta=None, encoding=None, priority=0,- dont_filter=False, errback=None, cb_kwargs=None, flags=None):- # type: (...) -> Request+ dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:"""Return a :class:`~.Request` instance to follow a link ``url``.It accepts the same arguments as ``Request.__init__`` method,@@ -184,8 +183,7 @@ class TextResponse(Response):def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,cookies=None, meta=None, encoding=None, priority=0,dont_filter=False, errback=None, cb_kwargs=None, flags=None,- css=None, xpath=None):- # type: (...) -> Generator[Request, None, None]+ css=None, xpath=None) -> Generator[Request, None, None]:"""A generator that produces :class:`~.Request` instances to follow alllinks in ``urls``. It accepts the same arguments as the :class:`~.Request`'sdiff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 8d7e5a602..431bd76dc 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -1,7 +1,7 @@import loggingimport pprintfrom collections import defaultdict, deque-from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast+from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, castfrom twisted.internet.defer import Deferred@@ -19,7 +19,7 @@ class MiddlewareManager:component_name = 'foo middleware'- def __init__(self, *middlewares):+ def __init__(self, *middlewares: Any) -> None:self.middlewares = middlewares# Only process_spider_output and process_spider_exception can be None.# Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.diff --git a/tox.ini b/tox.iniindex eee99cb2d..4d0f0291b 100644--- a/tox.ini+++ b/tox.ini@@ -38,7 +38,7 @@ install_command =basepython = python3deps =lxml-stubs==0.2.0- mypy==0.982+ mypy==0.991types-attrs==19.1.0types-pyOpenSSL==21.0.0types-setuptools==57.0.0@@ -58,7 +58,7 @@ deps ={[testenv]deps}# Twisted[http2] is required to import some filesTwisted[http2]>=17.9.0- flake8==5.0.4+ flake8==6.0.0commands =flake8 {posargs:docs scrapy tests}@@ -67,7 +67,7 @@ commands =basepython = python3.8deps ={[testenv:extra-deps]deps}- pylint==2.15.3+ pylint==2.15.6commands =pylint conftest.py docs extras scrapy setup.py tests