Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactor curl_to_request_kwargs #5001 Co-authored-by: alkazaz [email protected] Co-authored-by: swill [email protected] Co-authored-by: lerjevik [email protected] Co-authored-by: aljica [email protected]
Djiar3894ebb1497b
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex 6660b9dc0..d8b3deaa1 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments:curl_parser.add_argument(*argument, action='store_true')+def _parse_headers_and_cookies(parsed_args):+ headers = []+ cookies = {}+ for header in parsed_args.headers or ():+ name, val = header.split(':', 1)+ name = name.strip()+ val = val.strip()+ if name.title() == 'Cookie':+ for name, morsel in SimpleCookie(val).items():+ cookies[name] = morsel.value+ else:+ headers.append((name, val))++ if parsed_args.auth:+ user, password = parsed_args.auth.split(':', 1)+ headers.append(('Authorization', basic_auth_header(user, password)))++ return headers, cookies++def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):"""Convert a cURL command syntax to Request kwargs.@@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):result = {'method': method.upper(), 'url': url}- headers = []- cookies = {}- for header in parsed_args.headers or ():- name, val = header.split(':', 1)- name = name.strip()- val = val.strip()- if name.title() == 'Cookie':- for name, morsel in SimpleCookie(val).items():- cookies[name] = morsel.value- else:- headers.append((name, val))-- if parsed_args.auth:- user, password = parsed_args.auth.split(':', 1)- headers.append(('Authorization', basic_auth_header(user, password)))+ headers, cookies = _parse_headers_and_cookies(parsed_args)if headers:result['headers'] = headers