Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Refactor curl_to_request_kwargs #5001 Co-authored-by: alkazaz [email protected] Co-authored-by: swill [email protected] Co-authored-by: lerjevik [email protected] Co-authored-by: aljica [email protected]

Djiar3894ebb1497b
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 6660b9dc0..d8b3deaa1 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments:
curl_parser.add_argument(*argument, action='store_true')
+def _parse_headers_and_cookies(parsed_args):
+ headers = []
+ cookies = {}
+ for header in parsed_args.headers or ():
+ name, val = header.split(':', 1)
+ name = name.strip()
+ val = val.strip()
+ if name.title() == 'Cookie':
+ for name, morsel in SimpleCookie(val).items():
+ cookies[name] = morsel.value
+ else:
+ headers.append((name, val))
+
+ if parsed_args.auth:
+ user, password = parsed_args.auth.split(':', 1)
+ headers.append(('Authorization', basic_auth_header(user, password)))
+
+ return headers, cookies
+
+
def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
"""Convert a cURL command syntax to Request kwargs.
@@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
result = {'method': method.upper(), 'url': url}
- headers = []
- cookies = {}
- for header in parsed_args.headers or ():
- name, val = header.split(':', 1)
- name = name.strip()
- val = val.strip()
- if name.title() == 'Cookie':
- for name, morsel in SimpleCookie(val).items():
- cookies[name] = morsel.value
- else:
- headers.append((name, val))
-
- if parsed_args.auth:
- user, password = parsed_args.auth.split(':', 1)
- headers.append(('Authorization', basic_auth_header(user, password)))
+ headers, cookies = _parse_headers_and_cookies(parsed_args)
if headers:
result['headers'] = headers