Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into typing-request-response-cls

Andrey Rakhmatullin1268b2330419
diff --cc scrapy/http/request/__init__.py
index e7fdd5d54,77149333c..04589dd37
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@@ -20,10 -20,9 +20,11 @@@ from typing import
NoReturn,
Optional,
Tuple,
+ Type,
+ TypedDict,
+ TypeVar,
Union,
- cast,
+ overload,
)
from w3lib.url import safe_url_string
@@@ -36,13 -35,21 +37,24 @@@ from scrapy.utils.trackref import objec
from scrapy.utils.url import escape_ajax
if TYPE_CHECKING:
- # typing.Self requires Python 3.11
- from typing_extensions import Self
+ # typing.NotRequired and typing.Self require Python 3.11
+ from typing_extensions import NotRequired, Self
+
+
+ class VerboseCookie(TypedDict):
+ name: str
+ value: str
+ domain: NotRequired[str]
+ path: NotRequired[str]
+ secure: NotRequired[bool]
+
+
+ CookiesT = Union[Dict[str, str], List[VerboseCookie]]
+RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
+
+
def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
"""When assigned to the ``callback`` parameter of
:class:`~scrapy.http.Request`, it indicates that the request is not meant
diff --cc scrapy/http/request/json_request.py
index 59b11c692,405c0b9d0..057a4f897
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@@ -10,13 -8,9 +10,13 @@@ from __future__ import annotation
import copy
import json
import warnings
- from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload
-from typing import Any, Dict, Optional, Tuple
++from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload
+
+from scrapy.http.request import Request, RequestTypeVar
-from scrapy.http.request import Request
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
class JsonRequest(Request):
@@@ -47,31 -41,21 +47,31 @@@
)
@property
- def dumps_kwargs(self) -> dict:
+ def dumps_kwargs(self) -> Dict[str, Any]:
return self._dumps_kwargs
- def replace(self, *args: Any, **kwargs: Any) -> Request:
+ @overload
+ def replace(
+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+ ) -> RequestTypeVar: ...
+
+ @overload
+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
+
+ def replace(
+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+ ) -> Request:
body_passed = kwargs.get("body", None) is not None
- data = kwargs.pop("data", None)
- data_passed = data is not None
+ data: Any = kwargs.pop("data", None)
+ data_passed: bool = data is not None
if body_passed and data_passed:
warnings.warn("Both body and data passed. data will be ignored")
elif not body_passed and data_passed:
kwargs["body"] = self._dumps(data)
- return super().replace(*args, **kwargs)
+ return super().replace(*args, cls=cls, **kwargs)
- def _dumps(self, data: dict) -> str:
+ def _dumps(self, data: Any) -> str:
"""Convert to JSON"""
return json.dumps(data, **self._dumps_kwargs)