Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Implement a request_to_curl function (#5892)

guillermo-bondonno441ac196e415
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 409ca2e52..6d8be991d 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -327,3 +327,34 @@ def _get_method(obj, name):
return getattr(obj, name)
except AttributeError:
raise ValueError(f"Method {name!r} not found in: {obj}")
+
+
+def request_to_curl(request: Request) -> str:
+ """
+ Converts a :class:`~scrapy.Request` object to a curl command.
+
+ :param :class:`~scrapy.Request`: Request object to be converted
+ :return: string containing the curl command
+ """
+ method = request.method
+
+ data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else ""
+
+ headers = " ".join(
+ f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items()
+ )
+
+ url = request.url
+ cookies = ""
+ if request.cookies:
+ if isinstance(request.cookies, dict):
+ cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items())
+ cookies = f"--cookie '{cookie}'"
+ elif isinstance(request.cookies, list):
+ cookie = "; ".join(
+ f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies
+ )
+ cookies = f"--cookie '{cookie}'"
+
+ curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip()
+ return " ".join(curl_cmd.split())
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 6ca272de1..e6d1abe3f 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,3 +1,4 @@
+import json
import unittest
import warnings
from hashlib import sha1
@@ -18,6 +19,7 @@ from scrapy.utils.request import (
request_authenticate,
request_fingerprint,
request_httprepr,
+ request_to_curl,
)
from scrapy.utils.test import get_crawler
@@ -666,5 +668,67 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
self.assertEqual(fingerprint, settings["FINGERPRINT"])
+class RequestToCurlTest(unittest.TestCase):
+ def _test_request(self, request_object, expected_curl_command):
+ curl_command = request_to_curl(request_object)
+ self.assertEqual(curl_command, expected_curl_command)
+
+ def test_get(self):
+ request_object = Request("https://www.example.com")
+ expected_curl_command = "curl -X GET https://www.example.com"
+ self._test_request(request_object, expected_curl_command)
+
+ def test_post(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_headers(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ headers={"Content-Type": "application/json", "Accept": "application/json"},
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ ' --data-raw \'{"foo": "bar"}\''
+ " -H 'Content-Type: application/json' -H 'Accept: application/json'"
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_cookies_dict(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ cookies={"foo": "bar"},
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_cookies_list(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ cookies=[{"foo": "bar"}],
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+ )
+ self._test_request(request_object, expected_curl_command)
+
+
if __name__ == "__main__":
unittest.main()