Scrapy, a fast high-level web crawling & scraping framework for Python.
Implement a request_to_curl function (#5892)
guillermo-bondonno441ac196e415
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 409ca2e52..6d8be991d 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -327,3 +327,34 @@ def _get_method(obj, name):return getattr(obj, name)except AttributeError:raise ValueError(f"Method {name!r} not found in: {obj}")+++def request_to_curl(request: Request) -> str:+ """+ Converts a :class:`~scrapy.Request` object to a curl command.++ :param :class:`~scrapy.Request`: Request object to be converted+ :return: string containing the curl command+ """+ method = request.method++ data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else ""++ headers = " ".join(+ f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items()+ )++ url = request.url+ cookies = ""+ if request.cookies:+ if isinstance(request.cookies, dict):+ cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items())+ cookies = f"--cookie '{cookie}'"+ elif isinstance(request.cookies, list):+ cookie = "; ".join(+ f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies+ )+ cookies = f"--cookie '{cookie}'"++ curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip()+ return " ".join(curl_cmd.split())diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 6ca272de1..e6d1abe3f 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -1,3 +1,4 @@+import jsonimport unittestimport warningsfrom hashlib import sha1@@ -18,6 +19,7 @@ from scrapy.utils.request import (request_authenticate,request_fingerprint,request_httprepr,+ request_to_curl,)from scrapy.utils.test import get_crawler@@ -666,5 +668,67 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):self.assertEqual(fingerprint, settings["FINGERPRINT"])+class RequestToCurlTest(unittest.TestCase):+ def _test_request(self, request_object, expected_curl_command):+ curl_command = request_to_curl(request_object)+ self.assertEqual(curl_command, expected_curl_command)++ def test_get(self):+ request_object = Request("https://www.example.com")+ expected_curl_command = "curl -X GET https://www.example.com"+ self._test_request(request_object, expected_curl_command)++ def test_post(self):+ request_object = Request(+ "https://www.httpbin.org/post",+ method="POST",+ body=json.dumps({"foo": "bar"}),+ )+ expected_curl_command = (+ 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''+ )+ self._test_request(request_object, expected_curl_command)++ def test_headers(self):+ request_object = Request(+ "https://www.httpbin.org/post",+ method="POST",+ headers={"Content-Type": "application/json", "Accept": "application/json"},+ body=json.dumps({"foo": "bar"}),+ )+ expected_curl_command = (+ "curl -X POST https://www.httpbin.org/post"+ ' --data-raw \'{"foo": "bar"}\''+ " -H 'Content-Type: application/json' -H 'Accept: application/json'"+ )+ self._test_request(request_object, expected_curl_command)++ def test_cookies_dict(self):+ request_object = Request(+ "https://www.httpbin.org/post",+ method="POST",+ cookies={"foo": "bar"},+ body=json.dumps({"foo": "bar"}),+ )+ expected_curl_command = (+ "curl -X POST https://www.httpbin.org/post"+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"+ )+ self._test_request(request_object, expected_curl_command)++ def test_cookies_list(self):+ request_object = Request(+ "https://www.httpbin.org/post",+ method="POST",+ cookies=[{"foo": "bar"}],+ body=json.dumps({"foo": "bar"}),+ )+ expected_curl_command = (+ "curl -X POST https://www.httpbin.org/post"+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"+ )+ self._test_request(request_object, expected_curl_command)++if __name__ == "__main__":unittest.main()