Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactor request to/from dict (#5130)
Eugenio Lacuestacec36a928464
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 500781c05..73b5a858f 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -26,10 +26,6 @@ Request objects.. autoclass:: Request- A :class:`Request` object represents an HTTP request, which is usually- generated in the Spider and executed by the Downloader, and thus generating- a :class:`Response`.-:param url: the URL of this requestIf the URL is invalid, a :exc:`ValueError` exception is raised.@@ -205,6 +201,8 @@ Request objects``failure.request.cb_kwargs`` in the request's errback. For more information,see :ref:`errback-cb_kwargs`.+ .. autoattribute:: Request.attributes+.. method:: Request.copy()Return a new Request which is a copy of this Request. See also:@@ -220,6 +218,15 @@ Request objects.. automethod:: from_curl+ .. automethod:: to_dict+++Other functions related to requests+-----------------------------------++.. autofunction:: scrapy.utils.request.request_from_dict++.. _topics-request-response-ref-request-callback-arguments:Passing additional data to callback functions@@ -642,6 +649,8 @@ dealing with JSON requests.data into JSON format.:type dumps_kwargs: dict+ .. autoattribute:: JsonRequest.attributes+JsonRequest usage example-------------------------diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 498f1b052..ad884feac 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -4,17 +4,37 @@ requests in Scrapy.See documentation in docs/topics/request-response.rst"""+import inspect+from typing import Optional, Tuple+from w3lib.url import safe_url_string+import scrapy+from scrapy.http.common import obsolete_setterfrom scrapy.http.headers import Headers+from scrapy.utils.curl import curl_to_request_kwargsfrom scrapy.utils.python import to_bytesfrom scrapy.utils.trackref import object_reffrom scrapy.utils.url import escape_ajax-from scrapy.http.common import obsolete_setter-from scrapy.utils.curl import curl_to_request_kwargsclass Request(object_ref):+ """Represents an HTTP request, which is usually generated in a Spider and+ executed by the Downloader, thus generating a :class:`Response`.+ """++ attributes: Tuple[str, ...] = (+ "url", "callback", "method", "headers", "body",+ "cookies", "meta", "encoding", "priority",+ "dont_filter", "errback", "flags", "cb_kwargs",+ )+ """A tuple of :class:`str` objects containing the name of all public+ attributes of the class that are also keyword parameters of the+ ``__init__`` method.++ Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and+ :func:`~scrapy.utils.request.request_from_dict`.+ """def __init__(self, url, callback=None, method='GET', headers=None, body=None,cookies=None, meta=None, encoding='utf-8', priority=0,@@ -99,11 +119,8 @@ class Request(object_ref):return self.replace()def replace(self, *args, **kwargs):- """Create a new Request with the same attributes except for those- given new values.- """- for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',- 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:+ """Create a new Request with the same attributes except for those given new values"""+ for x in self.attributes:kwargs.setdefault(x, getattr(self, x))cls = kwargs.pop('cls', self.__class__)return cls(*args, **kwargs)@@ -136,8 +153,43 @@ class Request(object_ref):To translate a cURL command into a Scrapy request,you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.-- """+ """request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)request_kwargs.update(kwargs)return cls(**request_kwargs)++ def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:+ """Return a dictionary containing the Request's data.++ Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.++ If a spider is given, this method will try to find out the name of the spider methods used as callback+ and errback and include them in the output dict, raising an exception if they cannot be found.+ """+ d = {+ "url": self.url, # urls are safe (safe_string_url)+ "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,+ "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,+ "headers": dict(self.headers),+ }+ for attr in self.attributes:+ d.setdefault(attr, getattr(self, attr))+ if type(self) is not Request:+ d["_class"] = self.__module__ + '.' + self.__class__.__name__+ return d+++def _find_method(obj, func):+ """Helper function for Request.to_dict"""+ # Only instance methods contain ``__func__``+ if obj and hasattr(func, '__func__'):+ members = inspect.getmembers(obj, predicate=inspect.ismethod)+ for name, obj_func in members:+ # We need to use __func__ to access the original function object because instance+ # method objects are generated each time attribute is retrieved from instance.+ #+ # Reference: The standard type hierarchy+ # https://docs.python.org/3/reference/datamodel.html+ if obj_func.__func__ is func.__func__:+ return name+ raise ValueError(f"Function {func} is not an instance method in: {obj}")diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.pyindex eae3f9f6b..04e80d897 100644--- a/scrapy/http/request/json_request.py+++ b/scrapy/http/request/json_request.py@@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rstimport copyimport jsonimport warnings+from typing import Tuplefrom scrapy.http.request import Requestfrom scrapy.utils.deprecate import create_deprecated_classclass JsonRequest(Request):++ attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)+def __init__(self, *args, **kwargs):dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))dumps_kwargs.setdefault('sort_keys', True)@@ -36,6 +40,10 @@ class JsonRequest(Request):self.headers.setdefault('Content-Type', 'application/json')self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')+ @property+ def dumps_kwargs(self):+ return self._dumps_kwargs+def replace(self, *args, **kwargs):body_passed = kwargs.get('body', None) is not Nonedata = kwargs.pop('data', None)diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 16f7bf4b6..dff9b1350 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -9,7 +9,7 @@ import picklefrom queuelib import queuefrom scrapy.utils.deprecate import create_deprecated_class-from scrapy.utils.reqser import request_to_dict, request_from_dict+from scrapy.utils.request import request_from_dictdef _with_mkdir(queue_class):@@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class):return cls(crawler, key)def push(self, request):- request = request_to_dict(request, self.spider)+ request = request.to_dict(spider=self.spider)return super().push(request)def pop(self):request = super().pop()if not request:return None- return request_from_dict(request, self.spider)+ return request_from_dict(request, spider=self.spider)def peek(self):"""Returns the next object to be returned by :meth:`pop`,@@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class):request = super().peek()if not request:return None- return request_from_dict(request, self.spider)+ return request_from_dict(request, spider=self.spider)return ScrapyRequestQueuediff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.pyindex d38b1bc4d..c254b9f82 100644--- a/scrapy/utils/reqser.py+++ b/scrapy/utils/reqser.py@@ -1,95 +1,22 @@-"""-Helper functions for serializing (and deserializing) requests.-"""-import inspect+import warnings+from typing import Optional-from scrapy.http import Request-from scrapy.utils.python import to_unicode-from scrapy.utils.misc import load_object+import scrapy+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.request import request_from_dict as _from_dict-def request_to_dict(request, spider=None):- """Convert Request object to a dict.+warnings.warn(+ ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method"+ " and/or scrapy.utils.request.request_from_dict instead"),+ category=ScrapyDeprecationWarning,+ stacklevel=2,+)- If a spider is given, it will try to find out the name of the spider method- used in the callback and store that as the callback.- """- cb = request.callback- if callable(cb):- cb = _find_method(spider, cb)- eb = request.errback- if callable(eb):- eb = _find_method(spider, eb)- d = {- 'url': to_unicode(request.url), # urls should be safe (safe_string_url)- 'callback': cb,- 'errback': eb,- 'method': request.method,- 'headers': dict(request.headers),- 'body': request.body,- 'cookies': request.cookies,- 'meta': request.meta,- '_encoding': request._encoding,- 'priority': request.priority,- 'dont_filter': request.dont_filter,- 'flags': request.flags,- 'cb_kwargs': request.cb_kwargs,- }- if type(request) is not Request:- d['_class'] = request.__module__ + '.' + request.__class__.__name__- return d+def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict:+ return request.to_dict(spider=spider)-def request_from_dict(d, spider=None):- """Create Request object from a dict.- If a spider is given, it will try to resolve the callbacks looking at the- spider for methods with the same name.- """- cb = d['callback']- if cb and spider:- cb = _get_method(spider, cb)- eb = d['errback']- if eb and spider:- eb = _get_method(spider, eb)- request_cls = load_object(d['_class']) if '_class' in d else Request- return request_cls(- url=to_unicode(d['url']),- callback=cb,- errback=eb,- method=d['method'],- headers=d['headers'],- body=d['body'],- cookies=d['cookies'],- meta=d['meta'],- encoding=d['_encoding'],- priority=d['priority'],- dont_filter=d['dont_filter'],- flags=d.get('flags'),- cb_kwargs=d.get('cb_kwargs'),- )---def _find_method(obj, func):- # Only instance methods contain ``__func__``- if obj and hasattr(func, '__func__'):- members = inspect.getmembers(obj, predicate=inspect.ismethod)- for name, obj_func in members:- # We need to use __func__ to access the original- # function object because instance method objects- # are generated each time attribute is retrieved from- # instance.- #- # Reference: The standard type hierarchy- # https://docs.python.org/3/reference/datamodel.html- if obj_func.__func__ is func.__func__:- return name- raise ValueError(f"Function {func} is not an instance method in: {obj}")---def _get_method(obj, name):- name = str(name)- try:- return getattr(obj, name)- except AttributeError:- raise ValueError(f"Method {name!r} not found in: {obj}")+def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request":+ return _from_dict(d, spider=spider)diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 541368423..57dcc5f2c 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -11,8 +11,9 @@ from weakref import WeakKeyDictionaryfrom w3lib.http import basic_auth_headerfrom w3lib.url import canonicalize_url-from scrapy.http import Request+from scrapy import Request, Spiderfrom scrapy.utils.httpobj import urlparse_cached+from scrapy.utils.misc import load_objectfrom scrapy.utils.python import to_bytes, to_unicode@@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]:if referrer is None:return referrerreturn to_unicode(referrer, errors='replace')+++def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:+ """Create a :class:`~scrapy.Request` object from a dict.++ If a spider is given, it will try to resolve the callbacks looking at the+ spider for methods with the same name.+ """+ request_cls = load_object(d["_class"]) if "_class" in d else Request+ kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}+ if d.get("callback") and spider:+ kwargs["callback"] = _get_method(spider, d["callback"])+ if d.get("errback") and spider:+ kwargs["errback"] = _get_method(spider, d["errback"])+ return request_cls(**kwargs)+++def _get_method(obj, name):+ """Helper function for request_from_dict"""+ name = str(name)+ try:+ return getattr(obj, name)+ except AttributeError:+ raise ValueError(f"Method {name!r} not found in: {obj}")diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.pysimilarity index 68%rename from tests/test_utils_reqser.pyrename to tests/test_request_dict.pyindex ee68cf6b1..5bdcb975b 100644--- a/tests/test_utils_reqser.py+++ b/tests/test_request_dict.py@@ -1,8 +1,16 @@+import sysimport unittest+import warnings+from contextlib import suppress-from scrapy.http import Request, FormRequest-from scrapy.spiders import Spider-from scrapy.utils.reqser import request_to_dict, request_from_dict+from scrapy import Spider, Request+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.http import FormRequest, JsonRequest+from scrapy.utils.request import request_from_dict+++class CustomRequest(Request):+ passclass RequestSerializationTest(unittest.TestCase):@@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase):priority=20,meta={'a': 'b'},cb_kwargs={'k': 'v'},- flags=['testFlag'])+ flags=['testFlag'],+ )self._assert_serializes_ok(r, spider=self.spider)def test_latin1_body(self):@@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase):self._assert_serializes_ok(r)def _assert_serializes_ok(self, request, spider=None):- d = request_to_dict(request, spider=spider)+ d = request.to_dict(spider=spider)request2 = request_from_dict(d, spider=spider)self._assert_same_request(request, request2)@@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase):self.assertEqual(r1.cookies, r2.cookies)self.assertEqual(r1.meta, r2.meta)self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)+ self.assertEqual(r1.encoding, r2.encoding)self.assertEqual(r1._encoding, r2._encoding)self.assertEqual(r1.priority, r2.priority)self.assertEqual(r1.dont_filter, r2.dont_filter)self.assertEqual(r1.flags, r2.flags)+ if isinstance(r1, JsonRequest):+ self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs)def test_request_class(self):- r = FormRequest("http://www.example.com")- self._assert_serializes_ok(r, spider=self.spider)- r = CustomRequest("http://www.example.com")- self._assert_serializes_ok(r, spider=self.spider)+ r1 = FormRequest("http://www.example.com")+ self._assert_serializes_ok(r1, spider=self.spider)+ r2 = CustomRequest("http://www.example.com")+ self._assert_serializes_ok(r2, spider=self.spider)+ r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})+ self._assert_serializes_ok(r3, spider=self.spider)def test_callback_serialization(self):r = Request("http://www.example.com", callback=self.spider.parse_item,@@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase):callback=self.spider.parse_item_reference,errback=self.spider.handle_error_reference)self._assert_serializes_ok(r, spider=self.spider)- request_dict = request_to_dict(r, self.spider)+ request_dict = r.to_dict(spider=self.spider)self.assertEqual(request_dict['callback'], 'parse_item_reference')self.assertEqual(request_dict['errback'], 'handle_error_reference')@@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase):callback=self.spider._TestSpider__parse_item_reference,errback=self.spider._TestSpider__handle_error_reference)self._assert_serializes_ok(r, spider=self.spider)- request_dict = request_to_dict(r, self.spider)+ request_dict = r.to_dict(spider=self.spider)self.assertEqual(request_dict['callback'],'_TestSpider__parse_item_reference')self.assertEqual(request_dict['errback'],@@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase):def test_unserializable_callback1(self):r = Request("http://www.example.com", callback=lambda x: x)- self.assertRaises(ValueError, request_to_dict, r)- self.assertRaises(ValueError, request_to_dict, r, spider=self.spider)+ self.assertRaises(ValueError, r.to_dict, spider=self.spider)def test_unserializable_callback2(self):r = Request("http://www.example.com", callback=self.spider.parse_item)- self.assertRaises(ValueError, request_to_dict, r)+ self.assertRaises(ValueError, r.to_dict, spider=None)def test_unserializable_callback3(self):"""Parser method is removed or replaced dynamically."""class MySpider(Spider):-name = 'my_spider'def parse(self, response):@@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase):spider = MySpider()r = Request("http://www.example.com", callback=spider.parse)setattr(spider, 'parse', None)- self.assertRaises(ValueError, request_to_dict, r, spider=spider)+ self.assertRaises(ValueError, r.to_dict, spider=spider)++ def test_callback_not_available(self):+ """Callback method is not available in the spider passed to from_dict"""+ spider = TestSpiderDelegation()+ r = Request("http://www.example.com", callback=spider.delegated_callback)+ d = r.to_dict(spider=spider)+ self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))+++class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):+ def _assert_serializes_ok(self, request, spider=None):+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ with suppress(KeyError):+ del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning++ from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict++ request_copy = _from_dict(_to_dict(request, spider), spider)+ self._assert_same_request(request, request_copy)++ self.assertEqual(len(caught), 1)+ self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))+ self.assertEqual(+ "Module scrapy.utils.reqser is deprecated, please use request.to_dict method"+ " and/or scrapy.utils.request.request_from_dict instead",+ str(caught[0].message),+ )class TestSpiderMixin:@@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin):def __parse_item_private(self, response):pass---class CustomRequest(Request):- pass