🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor: Optimizations to CLI
Karim shoair29e485bdf2bc
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 9ea0ed0..ad5446f 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -3,7 +3,7 @@ from subprocess import check_outputfrom sys import executable as python_executablefrom scrapling.core.utils import log-from scrapling.core.shell import Convertor, _CookieParser+from scrapling.core.shell import Convertor, _CookieParser, _ParseHeadersfrom scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcherfrom orjson import loads as json_loads, JSONDecodeError@@ -22,31 +22,6 @@ def run_command(cmd, line):# I meant to not use try except here-def parse_headers(header_strings):- """Parse header strings into a dictionary"""- headers = {}- for header in header_strings:- if ":" in header:- key, value = header.split(":", 1)- headers[key.strip()] = value.strip()- else:- log.warning(f"Invalid header format '{header}', should be 'Key: Value'")- return headers---def parse_cookies(cookie_string):- """Parse cookie string into a dictionary"""- if not cookie_string:- return {}-- try:- cookies = {key: value for key, value in _CookieParser(cookie_string)}- except Exception as e:- raise ValueError(f"Could not parse cookies '{cookie_string}': {e}")-- return cookies--def parse_json_data(json_string):"""Parse JSON string into a Python object"""if not json_string:@@ -140,8 +115,13 @@ def shell(code, level):def parse_extract_arguments(headers, cookies, params, json=None):"""Parse arguments for extract command"""- parsed_headers = parse_headers(headers)- parsed_cookies = parse_cookies(cookies)+ parsed_headers, parsed_cookies = _ParseHeaders(headers)+ for key, value in _CookieParser(cookies):+ try:+ parsed_cookies[key] = value+ except Exception as e:+ raise ValueError(f"Could not parse cookies '{cookies}': {e}")+parsed_json = parse_json_data(json)parsed_params = {}for param in params:@@ -673,7 +653,7 @@ def fetch("""# Parse parameters- parsed_headers = parse_headers(extra_headers)+ parsed_headers, _ = _ParseHeaders(extra_headers, False)# Build request argumentskwargs = {@@ -821,7 +801,7 @@ def stealthy_fetch("""# Parse parameters- parsed_headers = parse_headers(extra_headers)+ parsed_headers, _ = _ParseHeaders(extra_headers, False)# Build request argumentskwargs = {diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 63ec304..4f966c0 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -73,6 +73,46 @@ def _CookieParser(cookie_string):yield key, morsel.value+def _ParseHeaders(+ header_lines: List[str], parse_cookies: bool = True+) -> Tuple[Dict[str, str], Dict[str, str]]:+ """Parses headers into separate header and cookie dictionaries."""+ header_dict = dict()+ cookie_dict = dict()++ for header_line in header_lines:+ if ":" not in header_line:+ if header_line.endswith(";"):+ header_key = header_line[:-1].strip()+ header_value = ""+ header_dict[header_key] = header_value+ else:+ raise ValueError(+ f"Could not parse header without colon: '{header_line}'."+ )+ else:+ header_key, header_value = header_line.split(":", 1)+ header_key = header_key.strip()+ header_value = header_value.strip()++ if parse_cookies:+ if header_key.lower() == "cookie":+ try:+ cookie_dict = {+ key: value for key, value in _CookieParser(header_value)+ }+ except Exception as e:+ raise ValueError(+ f"Could not parse cookie string from header '{header_value}': {e}"+ )+ else:+ header_dict[header_key] = header_value+ else:+ header_dict[header_key] = header_value++ return header_dict, cookie_dict++# Suppress exit on error to handle parsing errors gracefullyclass NoExitArgumentParser(ArgumentParser):def error(self, message):@@ -142,41 +182,6 @@ class CurlParser:self._supported_methods = ("get", "post", "put", "delete")# --- Helper Functions ---- @staticmethod- def parse_headers(header_lines: List[str]) -> Tuple[Dict[str, str], Dict[str, str]]:- """Parses -H headers into separate header and cookie dictionaries."""- header_dict = dict()- cookie_dict = dict()-- for header_line in header_lines:- if ":" not in header_line:- if header_line.endswith(";"):- header_key = header_line[:-1].strip()- header_value = ""- header_dict[header_key] = header_value- else:- log.warning(- f"Could not parse header without colon: '{header_line}', skipping."- )- continue- else:- header_key, header_value = header_line.split(":", 1)- header_key = header_key.strip()- header_value = header_value.strip()-- if header_key.lower() == "cookie":- try:- cookie_dict = {- key: value for key, value in _CookieParser(header_value)- }- except Exception as e:- raise ValueError(- f"Could not parse cookie string from -H '{header_value}': {e}"- )- else:- header_dict[header_key] = header_value-- return header_dict, cookie_dict# --- Main Parsing Logic ---def parse(self, curl_command: str) -> Optional[Request]:@@ -225,7 +230,7 @@ class CurlParser:):method = "post"- headers, cookies = self.parse_headers(parsed_args.header)+ headers, cookies = _ParseHeaders(parsed_args.header)if parsed_args.cookie:# We are focusing on the string format from DevTools.