Scrapy, a fast high-level web crawling & scraping framework for Python.
dumps_kwargs
kasun Herath8f1507a4a5de
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 02b853fc0..4e6f00bb0 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -514,9 +514,9 @@ JSONRequestThe JSONRequest class extends the base :class:`Request` class with functionality fordealing with JSON requests.-.. class:: JSONRequest(url, [data, ...])+.. class:: JSONRequest(url, [... data])- The :class:`JSONRequest` class adds a new argument to the constructor called data. The+ The :class:`JSONRequest` class adds two new argument to the constructor. Theremaining arguments are the same as for the :class:`Request` class and arenot documented here.@@ -529,6 +529,12 @@ dealing with JSON requests.set to ``'POST'`` automatically.:type data: JSON serializable object+ :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data+ into JSON format.+ :type dumps_kwargs: dict++.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps+JSONRequest usage example-------------------------diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.pyindex afc4356a3..7499610b9 100644--- a/scrapy/http/request/json_request.py+++ b/scrapy/http/request/json_request.py@@ -13,6 +13,7 @@ from scrapy.http.request import Requestclass JSONRequest(Request):def __init__(self, *args, **kwargs):+ dumps_kwargs = kwargs.pop('dumps_kwargs', {})body_passed = kwargs.get('body', None) is not Nonedata = kwargs.pop('data', None)data_passed = data is not None@@ -21,7 +22,7 @@ class JSONRequest(Request):warnings.warn('Both body and data passed. data will be ignored')elif not body_passed and data_passed:- kwargs['body'] = json.dumps(data, sort_keys=True)+ kwargs['body'] = self.dump(data, **dumps_kwargs)if 'method' not in kwargs:kwargs['method'] = 'POST'@@ -29,3 +30,21 @@ class JSONRequest(Request):super(JSONRequest, self).__init__(*args, **kwargs)self.headers.setdefault('Content-Type', 'application/json')self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')+ self._dumps_kwargs = dumps_kwargs++ def replace(self, *args, **kwargs):+ body_passed = kwargs.get('body', None) is not None+ data = kwargs.pop('data', None)+ data_passed = data is not None++ if body_passed and data_passed:+ warnings.warn('Both body and data passed. data will be ignored')++ elif not body_passed and data_passed:+ kwargs['body'] = self.dump(data, **self._dumps_kwargs)++ return super(JSONRequest, self).replace(*args, **kwargs)++ def dump(self, data, **kwargs):+ """Convert to JSON """+ return json.dumps(data, sort_keys=True, **kwargs)diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 5eb655c12..6dcfa25da 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -14,6 +14,8 @@ if six.PY3:from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponsefrom scrapy.utils.python import to_bytes, to_native_str+from tests import mock+class RequestTest(unittest.TestCase):@@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest):def test_data(self):r1 = self.request_class(url="http://www.example.com/")self.assertEqual(r1.body, b'')- self.assertEqual(r1.method, 'GET')body = b'body'r2 = self.request_class(url="http://www.example.com/", body=body)self.assertEqual(r2.body, body)- self.assertEqual(r2.method, 'GET')data = {'name': 'value',}r3 = self.request_class(url="http://www.example.com/", data=data)self.assertEqual(r3.body, to_bytes(json.dumps(data)))- self.assertEqual(r3.method, 'POST')+ # empty datar4 = self.request_class(url="http://www.example.com/", data=[])self.assertEqual(r4.body, to_bytes(json.dumps([])))- self.assertEqual(r4.method, 'POST')+ def test_data_method(self):+ # data is not passed+ r1 = self.request_class(url="http://www.example.com/")+ self.assertEqual(r1.method, 'GET')++ body = b'body'+ r2 = self.request_class(url="http://www.example.com/", body=body)+ self.assertEqual(r2.method, 'GET')++ data = {+ 'name': 'value',+ }+ r3 = self.request_class(url="http://www.example.com/", data=data)+ self.assertEqual(r3.method, 'POST')++ # method passed explicitly+ r4 = self.request_class(url="http://www.example.com/", data=data, method='GET')+ self.assertEqual(r4.method, 'GET')++ r5 = self.request_class(url="http://www.example.com/", data=[])+ self.assertEqual(r5.method, 'POST')++ def test_body_data(self):+ """ passing both body and data should result a warning """+ body = b'body'+ data = {+ 'name': 'value',+ }with warnings.catch_warnings(record=True) as _warnings:r5 = self.request_class(url="http://www.example.com/", body=body, data=data)self.assertEqual(r5.body, body)@@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest):self.assertEqual(len(_warnings), 1)self.assertIn('data will be ignored', str(_warnings[0].message))+ def test_empty_body_data(self):+ """ passing any body value and data should result a warning """+ data = {+ 'name': 'value',+ }with warnings.catch_warnings(record=True) as _warnings:r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)self.assertEqual(r6.body, b'')@@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest):self.assertEqual(len(_warnings), 1)self.assertIn('data will be ignored', str(_warnings[0].message))+ def test_body_none_data(self):+ data = {+ 'name': 'value',+ }with warnings.catch_warnings(record=True) as _warnings:r7 = self.request_class(url="http://www.example.com/", body=None, data=data)self.assertEqual(r7.body, to_bytes(json.dumps(data)))self.assertEqual(r7.method, 'POST')self.assertEqual(len(_warnings), 0)+ def test_body_data_none(self):with warnings.catch_warnings(record=True) as _warnings:r8 = self.request_class(url="http://www.example.com/", body=None, data=None)self.assertEqual(r8.method, 'GET')self.assertEqual(len(_warnings), 0)+ def test_dumps_sort_keys(self):+ """ Test that sort_keys=True is passed to json.dumps by default """+ data = {+ 'name': 'value',+ }+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:+ self.request_class(url="http://www.example.com/", data=data)+ kwargs = mock_dumps.call_args[1]+ self.assertEqual(kwargs['sort_keys'], True)++ def test_dumps_kwargs(self):+ """ Test that dumps_kwargs are passed to json.dumps """+ data = {+ 'name': 'value',+ }+ dumps_kwargs = {+ 'ensure_ascii': True,+ 'allow_nan': True,+ }+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:+ self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs)+ kwargs = mock_dumps.call_args[1]+ self.assertEqual(kwargs['ensure_ascii'], True)+ self.assertEqual(kwargs['allow_nan'], True)++ def test_replace_data(self):+ data1 = {+ 'name1': 'value1',+ }+ data2 = {+ 'name2': 'value2',+ }+ r1 = self.request_class(url="http://www.example.com/", data=data1)+ r2 = r1.replace(data=data2)+ self.assertEqual(r2.body, to_bytes(json.dumps(data2)))++ def test_replace_sort_keys(self):+ """ Test that replace provides sort_keys=True to json.dumps """+ data1 = {+ 'name1': 'value1',+ }+ data2 = {+ 'name2': 'value2',+ }+ r1 = self.request_class(url="http://www.example.com/", data=data1)+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:+ r1.replace(data=data2)+ kwargs = mock_dumps.call_args[1]+ self.assertEqual(kwargs['sort_keys'], True)++ def test_replace_dumps_kwargs(self):+ """ Test that dumps_kwargs are provided json.dumps when replace is called """+ data1 = {+ 'name1': 'value1',+ }+ data2 = {+ 'name2': 'value2',+ }+ dumps_kwargs = {+ 'ensure_ascii': True,+ 'allow_nan': True,+ }+ r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs)+ with mock.patch('json.dumps', return_value=b'') as mock_dumps:+ r1.replace(data=data2)+ kwargs = mock_dumps.call_args[1]+ self.assertEqual(kwargs['ensure_ascii'], True)+ self.assertEqual(kwargs['allow_nan'], True)+def tearDown(self):warnings.resetwarnings()super(JSONRequestTest, self).tearDown()