Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

get_retry_request: add typing information

Adrián Chavesc0f3ca193873
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 8955c7e4f..5e49a284a 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,8 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
-import logging
-from inspect import isclass
+from logging import getLogger, Logger
+from typing import Optional, Union
from twisted.internet import defer
from twisted.internet.error import (
@@ -24,24 +24,26 @@ from twisted.internet.error import (
)
from twisted.web.client import ResponseFailed
-from scrapy.exceptions import NotConfigured
-from scrapy.utils.response import response_status_message
from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
+from scrapy.http.request import Request
+from scrapy.spiders import Spider
from scrapy.utils.python import global_object_name
+from scrapy.utils.response import response_status_message
-retry_logger = logging.getLogger(__name__)
+retry_logger = getLogger(__name__)
def get_retry_request(
- request,
+ request: Request,
*,
- spider,
- reason='unspecified',
- max_retry_times=None,
- priority_adjust=None,
- logger=retry_logger,
- stats_base_key='retry',
+ spider: Spider,
+ reason: Union[str, Exception] = 'unspecified',
+ max_retry_times: Optional[int] = None,
+ priority_adjust: Union[int, float, None] = None,
+ logger: Logger = retry_logger,
+ stats_base_key: str = 'retry',
):
"""
Returns a new :class:`~scrapy.Request` object to retry the specified
@@ -105,7 +107,7 @@ def get_retry_request(
priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
new_request.priority = request.priority + priority_adjust
- if isclass(reason):
+ if callable(reason):
reason = reason()
if isinstance(reason, Exception):
reason = global_object_name(reason.__class__)