Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Refactor request to/from dict (#5130)

Eugenio Lacuestacec36a928464
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 500781c05..73b5a858f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -26,10 +26,6 @@ Request objects
.. autoclass:: Request
- A :class:`Request` object represents an HTTP request, which is usually
- generated in the Spider and executed by the Downloader, and thus generating
- a :class:`Response`.
-
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
@@ -205,6 +201,8 @@ Request objects
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
+ .. autoattribute:: Request.attributes
+
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
@@ -220,6 +218,15 @@ Request objects
.. automethod:: from_curl
+ .. automethod:: to_dict
+
+
+Other functions related to requests
+-----------------------------------
+
+.. autofunction:: scrapy.utils.request.request_from_dict
+
+
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@@ -642,6 +649,8 @@ dealing with JSON requests.
data into JSON format.
:type dumps_kwargs: dict
+ .. autoattribute:: JsonRequest.attributes
+
JsonRequest usage example
-------------------------
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 498f1b052..ad884feac 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,17 +4,37 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
+import inspect
+from typing import Optional, Tuple
+
from w3lib.url import safe_url_string
+import scrapy
+from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
+from scrapy.utils.curl import curl_to_request_kwargs
from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
-from scrapy.http.common import obsolete_setter
-from scrapy.utils.curl import curl_to_request_kwargs
class Request(object_ref):
+ """Represents an HTTP request, which is usually generated in a Spider and
+ executed by the Downloader, thus generating a :class:`Response`.
+ """
+
+ attributes: Tuple[str, ...] = (
+ "url", "callback", "method", "headers", "body",
+ "cookies", "meta", "encoding", "priority",
+ "dont_filter", "errback", "flags", "cb_kwargs",
+ )
+ """A tuple of :class:`str` objects containing the name of all public
+ attributes of the class that are also keyword parameters of the
+ ``__init__`` method.
+
+ Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and
+ :func:`~scrapy.utils.request.request_from_dict`.
+ """
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
@@ -99,11 +119,8 @@ class Request(object_ref):
return self.replace()
def replace(self, *args, **kwargs):
- """Create a new Request with the same attributes except for those
- given new values.
- """
- for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
- 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
+ """Create a new Request with the same attributes except for those given new values"""
+ for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
@@ -136,8 +153,43 @@ class Request(object_ref):
To translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
-
- """
+ """
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
request_kwargs.update(kwargs)
return cls(**request_kwargs)
+
+ def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
+ """Return a dictionary containing the Request's data.
+
+ Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
+
+ If a spider is given, this method will try to find out the name of the spider methods used as callback
+ and errback and include them in the output dict, raising an exception if they cannot be found.
+ """
+ d = {
+ "url": self.url, # urls are safe (safe_string_url)
+ "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,
+ "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,
+ "headers": dict(self.headers),
+ }
+ for attr in self.attributes:
+ d.setdefault(attr, getattr(self, attr))
+ if type(self) is not Request:
+ d["_class"] = self.__module__ + '.' + self.__class__.__name__
+ return d
+
+
+def _find_method(obj, func):
+ """Helper function for Request.to_dict"""
+ # Only instance methods contain ``__func__``
+ if obj and hasattr(func, '__func__'):
+ members = inspect.getmembers(obj, predicate=inspect.ismethod)
+ for name, obj_func in members:
+ # We need to use __func__ to access the original function object because instance
+ # method objects are generated each time attribute is retrieved from instance.
+ #
+ # Reference: The standard type hierarchy
+ # https://docs.python.org/3/reference/datamodel.html
+ if obj_func.__func__ is func.__func__:
+ return name
+ raise ValueError(f"Function {func} is not an instance method in: {obj}")
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index eae3f9f6b..04e80d897 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst
import copy
import json
import warnings
+from typing import Tuple
from scrapy.http.request import Request
from scrapy.utils.deprecate import create_deprecated_class
class JsonRequest(Request):
+
+ attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
+
def __init__(self, *args, **kwargs):
dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
dumps_kwargs.setdefault('sort_keys', True)
@@ -36,6 +40,10 @@ class JsonRequest(Request):
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
+ @property
+ def dumps_kwargs(self):
+ return self._dumps_kwargs
+
def replace(self, *args, **kwargs):
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 16f7bf4b6..dff9b1350 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -9,7 +9,7 @@ import pickle
from queuelib import queue
from scrapy.utils.deprecate import create_deprecated_class
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy.utils.request import request_from_dict
def _with_mkdir(queue_class):
@@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class):
return cls(crawler, key)
def push(self, request):
- request = request_to_dict(request, self.spider)
+ request = request.to_dict(spider=self.spider)
return super().push(request)
def pop(self):
request = super().pop()
if not request:
return None
- return request_from_dict(request, self.spider)
+ return request_from_dict(request, spider=self.spider)
def peek(self):
"""Returns the next object to be returned by :meth:`pop`,
@@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class):
request = super().peek()
if not request:
return None
- return request_from_dict(request, self.spider)
+ return request_from_dict(request, spider=self.spider)
return ScrapyRequestQueue
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d38b1bc4d..c254b9f82 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,95 +1,22 @@
-"""
-Helper functions for serializing (and deserializing) requests.
-"""
-import inspect
+import warnings
+from typing import Optional
-from scrapy.http import Request
-from scrapy.utils.python import to_unicode
-from scrapy.utils.misc import load_object
+import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.request import request_from_dict as _from_dict
-def request_to_dict(request, spider=None):
- """Convert Request object to a dict.
+warnings.warn(
+ ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
+ " and/or scrapy.utils.request.request_from_dict instead"),
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+)
- If a spider is given, it will try to find out the name of the spider method
- used in the callback and store that as the callback.
- """
- cb = request.callback
- if callable(cb):
- cb = _find_method(spider, cb)
- eb = request.errback
- if callable(eb):
- eb = _find_method(spider, eb)
- d = {
- 'url': to_unicode(request.url), # urls should be safe (safe_string_url)
- 'callback': cb,
- 'errback': eb,
- 'method': request.method,
- 'headers': dict(request.headers),
- 'body': request.body,
- 'cookies': request.cookies,
- 'meta': request.meta,
- '_encoding': request._encoding,
- 'priority': request.priority,
- 'dont_filter': request.dont_filter,
- 'flags': request.flags,
- 'cb_kwargs': request.cb_kwargs,
- }
- if type(request) is not Request:
- d['_class'] = request.__module__ + '.' + request.__class__.__name__
- return d
+def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict:
+ return request.to_dict(spider=spider)
-def request_from_dict(d, spider=None):
- """Create Request object from a dict.
- If a spider is given, it will try to resolve the callbacks looking at the
- spider for methods with the same name.
- """
- cb = d['callback']
- if cb and spider:
- cb = _get_method(spider, cb)
- eb = d['errback']
- if eb and spider:
- eb = _get_method(spider, eb)
- request_cls = load_object(d['_class']) if '_class' in d else Request
- return request_cls(
- url=to_unicode(d['url']),
- callback=cb,
- errback=eb,
- method=d['method'],
- headers=d['headers'],
- body=d['body'],
- cookies=d['cookies'],
- meta=d['meta'],
- encoding=d['_encoding'],
- priority=d['priority'],
- dont_filter=d['dont_filter'],
- flags=d.get('flags'),
- cb_kwargs=d.get('cb_kwargs'),
- )
-
-
-def _find_method(obj, func):
- # Only instance methods contain ``__func__``
- if obj and hasattr(func, '__func__'):
- members = inspect.getmembers(obj, predicate=inspect.ismethod)
- for name, obj_func in members:
- # We need to use __func__ to access the original
- # function object because instance method objects
- # are generated each time attribute is retrieved from
- # instance.
- #
- # Reference: The standard type hierarchy
- # https://docs.python.org/3/reference/datamodel.html
- if obj_func.__func__ is func.__func__:
- return name
- raise ValueError(f"Function {func} is not an instance method in: {obj}")
-
-
-def _get_method(obj, name):
- name = str(name)
- try:
- return getattr(obj, name)
- except AttributeError:
- raise ValueError(f"Method {name!r} not found in: {obj}")
+def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request":
+ return _from_dict(d, spider=spider)
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 541368423..57dcc5f2c 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary
from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
-from scrapy.http import Request
+from scrapy import Request, Spider
from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
@@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]:
if referrer is None:
return referrer
return to_unicode(referrer, errors='replace')
+
+
+def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
+ """Create a :class:`~scrapy.Request` object from a dict.
+
+ If a spider is given, it will try to resolve the callbacks looking at the
+ spider for methods with the same name.
+ """
+ request_cls = load_object(d["_class"]) if "_class" in d else Request
+ kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
+ if d.get("callback") and spider:
+ kwargs["callback"] = _get_method(spider, d["callback"])
+ if d.get("errback") and spider:
+ kwargs["errback"] = _get_method(spider, d["errback"])
+ return request_cls(**kwargs)
+
+
+def _get_method(obj, name):
+ """Helper function for request_from_dict"""
+ name = str(name)
+ try:
+ return getattr(obj, name)
+ except AttributeError:
+ raise ValueError(f"Method {name!r} not found in: {obj}")
diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.py
similarity index 68%
rename from tests/test_utils_reqser.py
rename to tests/test_request_dict.py
index ee68cf6b1..5bdcb975b 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_request_dict.py
@@ -1,8 +1,16 @@
+import sys
import unittest
+import warnings
+from contextlib import suppress
-from scrapy.http import Request, FormRequest
-from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy import Spider, Request
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.http import FormRequest, JsonRequest
+from scrapy.utils.request import request_from_dict
+
+
+class CustomRequest(Request):
+ pass
class RequestSerializationTest(unittest.TestCase):
@@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase):
priority=20,
meta={'a': 'b'},
cb_kwargs={'k': 'v'},
- flags=['testFlag'])
+ flags=['testFlag'],
+ )
self._assert_serializes_ok(r, spider=self.spider)
def test_latin1_body(self):
@@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r)
def _assert_serializes_ok(self, request, spider=None):
- d = request_to_dict(request, spider=spider)
+ d = request.to_dict(spider=spider)
request2 = request_from_dict(d, spider=spider)
self._assert_same_request(request, request2)
@@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase):
self.assertEqual(r1.cookies, r2.cookies)
self.assertEqual(r1.meta, r2.meta)
self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
+ self.assertEqual(r1.encoding, r2.encoding)
self.assertEqual(r1._encoding, r2._encoding)
self.assertEqual(r1.priority, r2.priority)
self.assertEqual(r1.dont_filter, r2.dont_filter)
self.assertEqual(r1.flags, r2.flags)
+ if isinstance(r1, JsonRequest):
+ self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs)
def test_request_class(self):
- r = FormRequest("http://www.example.com")
- self._assert_serializes_ok(r, spider=self.spider)
- r = CustomRequest("http://www.example.com")
- self._assert_serializes_ok(r, spider=self.spider)
+ r1 = FormRequest("http://www.example.com")
+ self._assert_serializes_ok(r1, spider=self.spider)
+ r2 = CustomRequest("http://www.example.com")
+ self._assert_serializes_ok(r2, spider=self.spider)
+ r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
+ self._assert_serializes_ok(r3, spider=self.spider)
def test_callback_serialization(self):
r = Request("http://www.example.com", callback=self.spider.parse_item,
@@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase):
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
- request_dict = request_to_dict(r, self.spider)
+ request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict['callback'], 'parse_item_reference')
self.assertEqual(request_dict['errback'], 'handle_error_reference')
@@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase):
callback=self.spider._TestSpider__parse_item_reference,
errback=self.spider._TestSpider__handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
- request_dict = request_to_dict(r, self.spider)
+ request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict['callback'],
'_TestSpider__parse_item_reference')
self.assertEqual(request_dict['errback'],
@@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase):
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
- self.assertRaises(ValueError, request_to_dict, r)
- self.assertRaises(ValueError, request_to_dict, r, spider=self.spider)
+ self.assertRaises(ValueError, r.to_dict, spider=self.spider)
def test_unserializable_callback2(self):
r = Request("http://www.example.com", callback=self.spider.parse_item)
- self.assertRaises(ValueError, request_to_dict, r)
+ self.assertRaises(ValueError, r.to_dict, spider=None)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
-
name = 'my_spider'
def parse(self, response):
@@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase):
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
setattr(spider, 'parse', None)
- self.assertRaises(ValueError, request_to_dict, r, spider=spider)
+ self.assertRaises(ValueError, r.to_dict, spider=spider)
+
+ def test_callback_not_available(self):
+ """Callback method is not available in the spider passed to from_dict"""
+ spider = TestSpiderDelegation()
+ r = Request("http://www.example.com", callback=spider.delegated_callback)
+ d = r.to_dict(spider=spider)
+ self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
+
+
+class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):
+ def _assert_serializes_ok(self, request, spider=None):
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ with suppress(KeyError):
+ del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning
+
+ from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict
+
+ request_copy = _from_dict(_to_dict(request, spider), spider)
+ self._assert_same_request(request, request_copy)
+
+ self.assertEqual(len(caught), 1)
+ self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+ self.assertEqual(
+ "Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
+ " and/or scrapy.utils.request.request_from_dict instead",
+ str(caught[0].message),
+ )
class TestSpiderMixin:
@@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin):
def __parse_item_private(self, response):
pass
-
-
-class CustomRequest(Request):
- pass