Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add --data-raw to utils.curl and fix missing method with data (#4612)

Marc Hernández464f24f8c160
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 16639356e..67b22dbc5 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser):
curl_parser = CurlParser()
curl_parser.add_argument('url')
curl_parser.add_argument('-H', '--header', dest='headers', action='append')
-curl_parser.add_argument('-X', '--request', dest='method', default='get')
-curl_parser.add_argument('-d', '--data', dest='data')
+curl_parser.add_argument('-X', '--request', dest='method')
+curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
curl_parser.add_argument('-u', '--user', dest='auth')
@@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
if not parsed_url.scheme:
url = 'http://' + url
- result = {'method': parsed_args.method.upper(), 'url': url}
+ method = parsed_args.method or 'GET'
+
+ result = {'method': method.upper(), 'url': url}
headers = []
cookies = {}
@@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
result['cookies'] = cookies
if parsed_args.data:
result['body'] = parsed_args.data
+ if not parsed_args.method:
+ # if the "data" is specified but the "method" is not specified,
+ # the default method is 'POST'
+ result['method'] = 'POST'
return result
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 50e1bfd5f..299a51efe 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase):
}
self._test_command(curl_command, expected_result)
+ def test_post_data_raw(self):
+ curl_command = (
+ "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"
+ "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"
+ "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"
+ )
+ expected_result = {
+ "method": "POST",
+ "url": "https://www.example.org/",
+ "body": (
+ "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
+ "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
+ "Functions=%5B%5D")
+ }
+ self._test_command(curl_command, expected_result)
+
+ def test_explicit_get_with_data(self):
+ curl_command = 'curl httpbin.org/anything -X GET --data asdf'
+ expected_result = {
+ "method": "GET",
+ "url": "http://httpbin.org/anything",
+ "body": "asdf"
+ }
+ self._test_command(curl_command, expected_result)
+
def test_patch(self):
curl_command = (
'curl "https://example.com/api/fake" -u "username:password" -H "Ac'