Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

curl: add support for parsing -b,--cookie (#6684)

Matt Winter8c34e6d9a499
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index a563dc79a..a40ee8997 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -36,6 +36,7 @@ curl_parser = CurlParser()
curl_parser.add_argument("url")
curl_parser.add_argument("-H", "--header", dest="headers", action="append")
curl_parser.add_argument("-X", "--request", dest="method")
+curl_parser.add_argument("-b", "--cookie", dest="cookies", action="append")
curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction)
curl_parser.add_argument("-u", "--user", dest="auth")
@@ -68,6 +69,14 @@ def _parse_headers_and_cookies(
else:
headers.append((name, val))
+ for cookie_param in parsed_args.cookies or ():
+ # curl can treat this parameter as either "key=value; key2=value2" pairs, or a filename.
+ # Scrapy will only support key-value pairs.
+ if "=" not in cookie_param:
+ continue
+ for name, morsel in SimpleCookie(cookie_param).items():
+ cookies[name] = morsel.value
+
if parsed_args.auth:
user, password = parsed_args.auth.split(":", 1)
headers.append(("Authorization", basic_auth_header(user, password)))
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 1816db29b..5d99161bf 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -49,8 +49,8 @@ class CurlToRequestKwargsTest(unittest.TestCase):
"ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a"
"png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau"
"ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau"
- "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali"
- "ve' --compressed"
+ "ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_"
+ "gauges_unique_day=1'"
)
expected_result = {
"method": "GET",