Scrapy, a fast high-level web crawling & scraping framework for Python.
Add --data-raw to utils.curl and fix missing method with data (#4612)
Marc Hernández464f24f8c160
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex 16639356e..67b22dbc5 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser):curl_parser = CurlParser()curl_parser.add_argument('url')curl_parser.add_argument('-H', '--header', dest='headers', action='append')-curl_parser.add_argument('-X', '--request', dest='method', default='get')-curl_parser.add_argument('-d', '--data', dest='data')+curl_parser.add_argument('-X', '--request', dest='method')+curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')curl_parser.add_argument('-u', '--user', dest='auth')@@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):if not parsed_url.scheme:url = 'http://' + url- result = {'method': parsed_args.method.upper(), 'url': url}+ method = parsed_args.method or 'GET'++ result = {'method': method.upper(), 'url': url}headers = []cookies = {}@@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):result['cookies'] = cookiesif parsed_args.data:result['body'] = parsed_args.data+ if not parsed_args.method:+ # if the "data" is specified but the "method" is not specified,+ # the default method is 'POST'+ result['method'] = 'POST'return resultdiff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 50e1bfd5f..299a51efe 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase):}self._test_command(curl_command, expected_result)+ def test_post_data_raw(self):+ curl_command = (+ "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"+ "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"+ "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"+ )+ expected_result = {+ "method": "POST",+ "url": "https://www.example.org/",+ "body": (+ "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"+ "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"+ "Functions=%5B%5D")+ }+ self._test_command(curl_command, expected_result)++ def test_explicit_get_with_data(self):+ curl_command = 'curl httpbin.org/anything -X GET --data asdf'+ expected_result = {+ "method": "GET",+ "url": "http://httpbin.org/anything",+ "body": "asdf"+ }+ self._test_command(curl_command, expected_result)+def test_patch(self):curl_command = ('curl "https://example.com/api/fake" -u "username:password" -H "Ac'