Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

dumps_kwargs

kasun Herath8f1507a4a5de
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 02b853fc0..4e6f00bb0 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -514,9 +514,9 @@ JSONRequest
The JSONRequest class extends the base :class:`Request` class with functionality for
dealing with JSON requests.
-.. class:: JSONRequest(url, [data, ...])
+.. class:: JSONRequest(url, [... data])
- The :class:`JSONRequest` class adds a new argument to the constructor called data. The
+ The :class:`JSONRequest` class adds two new argument to the constructor. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@@ -529,6 +529,12 @@ dealing with JSON requests.
set to ``'POST'`` automatically.
:type data: JSON serializable object
+ :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data
+ into JSON format.
+ :type dumps_kwargs: dict
+
+.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
+
JSONRequest usage example
-------------------------
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index afc4356a3..7499610b9 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -13,6 +13,7 @@ from scrapy.http.request import Request
class JSONRequest(Request):
def __init__(self, *args, **kwargs):
+ dumps_kwargs = kwargs.pop('dumps_kwargs', {})
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)
data_passed = data is not None
@@ -21,7 +22,7 @@ class JSONRequest(Request):
warnings.warn('Both body and data passed. data will be ignored')
elif not body_passed and data_passed:
- kwargs['body'] = json.dumps(data, sort_keys=True)
+ kwargs['body'] = self.dump(data, **dumps_kwargs)
if 'method' not in kwargs:
kwargs['method'] = 'POST'
@@ -29,3 +30,21 @@ class JSONRequest(Request):
super(JSONRequest, self).__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
+ self._dumps_kwargs = dumps_kwargs
+
+ def replace(self, *args, **kwargs):
+ body_passed = kwargs.get('body', None) is not None
+ data = kwargs.pop('data', None)
+ data_passed = data is not None
+
+ if body_passed and data_passed:
+ warnings.warn('Both body and data passed. data will be ignored')
+
+ elif not body_passed and data_passed:
+ kwargs['body'] = self.dump(data, **self._dumps_kwargs)
+
+ return super(JSONRequest, self).replace(*args, **kwargs)
+
+ def dump(self, data, **kwargs):
+ """Convert to JSON """
+ return json.dumps(data, sort_keys=True, **kwargs)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5eb655c12..6dcfa25da 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -14,6 +14,8 @@ if six.PY3:
from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_native_str
+from tests import mock
+
class RequestTest(unittest.TestCase):
@@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest):
def test_data(self):
r1 = self.request_class(url="http://www.example.com/")
self.assertEqual(r1.body, b'')
- self.assertEqual(r1.method, 'GET')
body = b'body'
r2 = self.request_class(url="http://www.example.com/", body=body)
self.assertEqual(r2.body, body)
- self.assertEqual(r2.method, 'GET')
data = {
'name': 'value',
}
r3 = self.request_class(url="http://www.example.com/", data=data)
self.assertEqual(r3.body, to_bytes(json.dumps(data)))
- self.assertEqual(r3.method, 'POST')
+ # empty data
r4 = self.request_class(url="http://www.example.com/", data=[])
self.assertEqual(r4.body, to_bytes(json.dumps([])))
- self.assertEqual(r4.method, 'POST')
+ def test_data_method(self):
+ # data is not passed
+ r1 = self.request_class(url="http://www.example.com/")
+ self.assertEqual(r1.method, 'GET')
+
+ body = b'body'
+ r2 = self.request_class(url="http://www.example.com/", body=body)
+ self.assertEqual(r2.method, 'GET')
+
+ data = {
+ 'name': 'value',
+ }
+ r3 = self.request_class(url="http://www.example.com/", data=data)
+ self.assertEqual(r3.method, 'POST')
+
+ # method passed explicitly
+ r4 = self.request_class(url="http://www.example.com/", data=data, method='GET')
+ self.assertEqual(r4.method, 'GET')
+
+ r5 = self.request_class(url="http://www.example.com/", data=[])
+ self.assertEqual(r5.method, 'POST')
+
+ def test_body_data(self):
+ """ passing both body and data should result a warning """
+ body = b'body'
+ data = {
+ 'name': 'value',
+ }
with warnings.catch_warnings(record=True) as _warnings:
r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
self.assertEqual(r5.body, body)
@@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest):
self.assertEqual(len(_warnings), 1)
self.assertIn('data will be ignored', str(_warnings[0].message))
+ def test_empty_body_data(self):
+ """ passing any body value and data should result a warning """
+ data = {
+ 'name': 'value',
+ }
with warnings.catch_warnings(record=True) as _warnings:
r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)
self.assertEqual(r6.body, b'')
@@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest):
self.assertEqual(len(_warnings), 1)
self.assertIn('data will be ignored', str(_warnings[0].message))
+ def test_body_none_data(self):
+ data = {
+ 'name': 'value',
+ }
with warnings.catch_warnings(record=True) as _warnings:
r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
self.assertEqual(r7.body, to_bytes(json.dumps(data)))
self.assertEqual(r7.method, 'POST')
self.assertEqual(len(_warnings), 0)
+ def test_body_data_none(self):
with warnings.catch_warnings(record=True) as _warnings:
r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
self.assertEqual(r8.method, 'GET')
self.assertEqual(len(_warnings), 0)
+ def test_dumps_sort_keys(self):
+ """ Test that sort_keys=True is passed to json.dumps by default """
+ data = {
+ 'name': 'value',
+ }
+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+ self.request_class(url="http://www.example.com/", data=data)
+ kwargs = mock_dumps.call_args[1]
+ self.assertEqual(kwargs['sort_keys'], True)
+
+ def test_dumps_kwargs(self):
+ """ Test that dumps_kwargs are passed to json.dumps """
+ data = {
+ 'name': 'value',
+ }
+ dumps_kwargs = {
+ 'ensure_ascii': True,
+ 'allow_nan': True,
+ }
+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+ self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs)
+ kwargs = mock_dumps.call_args[1]
+ self.assertEqual(kwargs['ensure_ascii'], True)
+ self.assertEqual(kwargs['allow_nan'], True)
+
+ def test_replace_data(self):
+ data1 = {
+ 'name1': 'value1',
+ }
+ data2 = {
+ 'name2': 'value2',
+ }
+ r1 = self.request_class(url="http://www.example.com/", data=data1)
+ r2 = r1.replace(data=data2)
+ self.assertEqual(r2.body, to_bytes(json.dumps(data2)))
+
+ def test_replace_sort_keys(self):
+ """ Test that replace provides sort_keys=True to json.dumps """
+ data1 = {
+ 'name1': 'value1',
+ }
+ data2 = {
+ 'name2': 'value2',
+ }
+ r1 = self.request_class(url="http://www.example.com/", data=data1)
+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+ r1.replace(data=data2)
+ kwargs = mock_dumps.call_args[1]
+ self.assertEqual(kwargs['sort_keys'], True)
+
+ def test_replace_dumps_kwargs(self):
+ """ Test that dumps_kwargs are provided json.dumps when replace is called """
+ data1 = {
+ 'name1': 'value1',
+ }
+ data2 = {
+ 'name2': 'value2',
+ }
+ dumps_kwargs = {
+ 'ensure_ascii': True,
+ 'allow_nan': True,
+ }
+ r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs)
+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:
+ r1.replace(data=data2)
+ kwargs = mock_dumps.call_args[1]
+ self.assertEqual(kwargs['ensure_ascii'], True)
+ self.assertEqual(kwargs['allow_nan'], True)
+
def tearDown(self):
warnings.resetwarnings()
super(JSONRequestTest, self).tearDown()