Scrapy, a fast high-level web crawling & scraping framework for Python.
curl: add support for parsing -b,--cookie (#6684)
Matt Winter8c34e6d9a499
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex a563dc79a..a40ee8997 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -36,6 +36,7 @@ curl_parser = CurlParser()curl_parser.add_argument("url")curl_parser.add_argument("-H", "--header", dest="headers", action="append")curl_parser.add_argument("-X", "--request", dest="method")+curl_parser.add_argument("-b", "--cookie", dest="cookies", action="append")curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction)curl_parser.add_argument("-u", "--user", dest="auth")@@ -68,6 +69,14 @@ def _parse_headers_and_cookies(else:headers.append((name, val))+ for cookie_param in parsed_args.cookies or ():+ # curl can treat this parameter as either "key=value; key2=value2" pairs, or a filename.+ # Scrapy will only support key-value pairs.+ if "=" not in cookie_param:+ continue+ for name, morsel in SimpleCookie(cookie_param).items():+ cookies[name] = morsel.value+if parsed_args.auth:user, password = parsed_args.auth.split(":", 1)headers.append(("Authorization", basic_auth_header(user, password)))diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 1816db29b..5d99161bf 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -49,8 +49,8 @@ class CurlToRequestKwargsTest(unittest.TestCase):"ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a""png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau""ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau"- "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali"- "ve' --compressed"+ "ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_"+ "gauges_unique_day=1'")expected_result = {"method": "GET",