Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Type hints for Request subclasses

Eugenio Lacuesta479260dca012
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 04e80d897..dba3c3a82 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -8,9 +8,9 @@ See documentation in docs/topics/request-response.rst
import copy
import json
import warnings
-from typing import Tuple
+from typing import Optional, Tuple
-from scrapy.http.request import Request
+from scrapy.http.request import Request, RequestTypeVar
from scrapy.utils.deprecate import create_deprecated_class
@@ -18,8 +18,8 @@ class JsonRequest(Request):
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
- def __init__(self, *args, **kwargs):
- dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
+ def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
+ dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
dumps_kwargs.setdefault('sort_keys', True)
self._dumps_kwargs = dumps_kwargs
@@ -29,10 +29,8 @@ class JsonRequest(Request):
if body_passed and data_passed:
warnings.warn('Both body and data passed. data will be ignored')
-
elif not body_passed and data_passed:
kwargs['body'] = self._dumps(data)
-
if 'method' not in kwargs:
kwargs['method'] = 'POST'
@@ -41,23 +39,22 @@ class JsonRequest(Request):
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
@property
- def dumps_kwargs(self):
+ def dumps_kwargs(self) -> dict:
return self._dumps_kwargs
- def replace(self, *args, **kwargs):
+ def replace(self, *args, **kwargs) -> RequestTypeVar:
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)
data_passed = data is not None
if body_passed and data_passed:
warnings.warn('Both body and data passed. data will be ignored')
-
elif not body_passed and data_passed:
kwargs['body'] = self._dumps(data)
return super().replace(*args, **kwargs)
- def _dumps(self, data):
+ def _dumps(self, data: dict) -> str:
"""Convert to JSON """
return json.dumps(data, **self._dumps_kwargs)
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index c70912e49..06d98cea5 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
import xmlrpc.client as xmlrpclib
+from typing import Optional
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args
@@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
class XmlRpcRequest(Request):
- def __init__(self, *args, **kwargs):
- encoding = kwargs.get('encoding', None)
+ def __init__(self, *args, encoding: Optional[str] = None, **kwargs):
if 'body' not in kwargs and 'params' in kwargs:
kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
kwargs['body'] = xmlrpclib.dumps(**kw)