🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(cli): Adding two new commands (uncurl/curl2fetcher)
Karim shoair98ad50d5aa79
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 9e6eaab..481dda4 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -1,12 +1,10 @@import os-import subprocessimport sys+import subprocessfrom pathlib import Pathimport click-from scrapling.core.shell import CustomShell-def get_package_dir():return Path(os.path.dirname(__file__))@@ -73,6 +71,8 @@ def install(force):help="Log level (default: DEBUG)",)def shell(code, level):+ from scrapling.core.shell import CustomShell+console = CustomShell(code=code, log_level=level)console.start()diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex c3961e4..1b15aaa 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -1,36 +1,360 @@+# -*- coding: utf-8 -*-import os-import logging-import tempfile-import webbrowser+import json+from sys import stderrfrom functools import wraps+from http import cookies as Cookie+from collections import namedtuple+from shlex import split as shlex_split+from tempfile import mkstemp as make_temp_file+from urllib.parse import urlparse, urlunparse, parse_qsl+from argparse import ArgumentParser, SUPPRESS+from webbrowser import open as open_in_browser+from logging import (+ DEBUG,+ INFO,+ WARNING,+ ERROR,+ CRITICAL,+ FATAL,+ getLogger,+ getLevelName,+)from IPython.terminal.embed import InteractiveShellEmbedfrom scrapling import __version__from scrapling.core.utils import logfrom scrapling.parser import Adaptor, Adaptors-from scrapling.fetchers import Fetcher, AsyncFetcher, PlayWrightFetcher, StealthyFetcher-+from scrapling.core._types import List, Optional, Dict, Tuple, Any, Union+from scrapling.fetchers import (+ Fetcher,+ AsyncFetcher,+ PlayWrightFetcher,+ StealthyFetcher,+ Response,+)_known_logging_levels = {- "debug": logging.DEBUG,- "info": logging.INFO,- "warning": logging.WARNING,- "error": logging.ERROR,- "critical": logging.CRITICAL,- "fatal": logging.FATAL,+ "debug": DEBUG,+ "info": INFO,+ "warning": WARNING,+ "error": ERROR,+ "critical": CRITICAL,+ "fatal": FATAL,}+# Define the structure for parsed context - Simplified for Fetcher args+Request = namedtuple(+ "Request",+ [+ "method",+ "url",+ "params",+ "data", # Can be str, bytes, or dict (for urlencoded)+ "json_data", # Python object (dict/list) for JSON payload+ "headers",+ "cookies",+ "proxy",+ "follow_redirects", # Added for -L flag+ ],+)+++# Suppress exit on error to handle parsing errors gracefully+class NoExitArgumentParser(ArgumentParser):+ def error(self, message):+ log.error(f"Curl arguments parsing error: {message}")+ raise ValueError(f"Curl arguments parsing error: {message}")++ def exit(self, status=0, message=None):+ if message:+ log.error(f"Scrapling shell exited with status {status}: {message}")+ self._print_message(message, stderr)+ raise ValueError(+ f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}"+ )+++class CurlParser:+ """Builds the argument parser for relevant curl flags from DevTools."""++ def __init__(self):+ # We will use argparse parser to parse the curl command directly instead of regex+ # We will focus more on flags that will show up on curl commands copied from DevTools's network tab+ _parser = NoExitArgumentParser(add_help=False) # Disable default help+ # Basic curl arguments+ _parser.add_argument("curl_command_placeholder", nargs="?", help=SUPPRESS)+ _parser.add_argument("url")+ _parser.add_argument("-X", "--request", dest="method", default=None)+ _parser.add_argument("-H", "--header", action="append", default=[])+ _parser.add_argument(+ "-A", "--user-agent", help="Will be parsed from -H if present"+ ) # Note: DevTools usually includes this in -H++ # Data arguments (prioritizing types common from DevTools)+ _parser.add_argument("-d", "--data", default=None)+ _parser.add_argument(+ "--data-raw", default=None+ ) # Often used by browsers for JSON body+ _parser.add_argument("--data-binary", default=None)+ # Keep urlencode for completeness, though less common from browser copy/paste+ _parser.add_argument("--data-urlencode", action="append", default=[])+ _parser.add_argument(+ "-G", "--get", action="store_true"+ ) # Use GET and put data in URL++ # Proxy+ _parser.add_argument("-x", "--proxy", default=None)+ _parser.add_argument("-U", "--proxy-user", default=None) # Basic proxy auth++ # Connection/Security+ _parser.add_argument("-k", "--insecure", action="store_true")+ _parser.add_argument(+ "--compressed", action="store_true"+ ) # Very common from browsers++ # Other flags often included but may not map directly to request args+ _parser.add_argument("-i", "--include", action="store_true")+ _parser.add_argument("-s", "--silent", action="store_true")+ _parser.add_argument("-v", "--verbose", action="store_true")++ self.parser: NoExitArgumentParser = _parser+ self._supported_methods = ("get", "post", "put", "delete")++ # --- Helper Functions ---+ @staticmethod+ def parse_headers(header_lines: List[str]) -> Tuple[Dict[str, str], Dict[str, str]]:+ """Parses -H headers into separate header and cookie dictionaries."""+ header_dict = dict()+ cookie_dict = dict()++ for header_line in header_lines:+ if ":" not in header_line:+ if header_line.endswith(";"):+ header_key = header_line[:-1].strip()+ header_value = ""+ header_dict[header_key] = header_value+ else:+ log.warning(+ f"Could not parse header without colon: '{header_line}', skipping."+ )+ continue+ else:+ header_key, header_value = header_line.split(":", 1)+ header_key = header_key.strip()+ header_value = header_value.strip()++ if header_key.lower() == "cookie":+ try:+ cookie_parser = Cookie.SimpleCookie()+ cookie_parser.load(header_value)+ for key, morsel in cookie_parser.items():+ cookie_dict[key] = morsel.value+ except Exception as e:+ log.error(+ f"Could not parse cookie string '{header_value}': {e}"+ )+ else:+ header_dict[header_key] = header_value++ return header_dict, cookie_dict++ # --- Main Parsing Logic ---+ def parse(self, curl_command: str) -> Optional[Request]:+ """Parses the curl command string into a structured context for Fetcher."""++ clean_command = curl_command.strip().lstrip("curl").strip().replace("\\\n", " ")++ try:+ tokens = shlex_split(+ clean_command+ ) # Split the string using shell-like syntax+ except ValueError as e:+ log.error(f"Could not split command line: {e}")+ return None++ try:+ parsed_args, unknown = self.parser.parse_known_args(tokens)+ if unknown:+ log.warning(f"Ignored unknown curl arguments: {unknown}")++ except ValueError:+ return None++ except Exception as e:+ log.error(+ f"An unexpected error occurred during curl arguments parsing: {e}"+ )+ return None++ # --- Determine Method ---+ method = "get" # Default+ if parsed_args.get: # -G forces GET+ method = "get"++ elif parsed_args.method:+ method = parsed_args.method.strip().lower()++ # Infer POST if data is present (unless overridden by -X or -G)+ elif any(+ [+ parsed_args.data,+ parsed_args.data_raw,+ parsed_args.data_binary,+ parsed_args.data_urlencode,+ ]+ ):+ method = "post"++ headers, cookies = self.parse_headers(parsed_args.header)++ # --- Process Data Payload ---+ params = dict()+ data_payload: Union[str, bytes, Dict, None] = None+ json_payload: Optional[Any] = None++ # DevTools often uses --data-raw for JSON bodies+ # Precedence: --data-binary > --data-raw / -d > --data-urlencode+ if parsed_args.data_binary is not None:+ try:+ data_payload = parsed_args.data_binary.encode("utf-8")+ log.debug("Using data from --data-binary as bytes.")+ except Exception as e:+ log.warning(+ f"Could not encode binary data '{parsed_args.data_binary}' as bytes: {e}. Using raw string."+ )+ data_payload = parsed_args.data_binary # Fallback to string++ elif parsed_args.data_raw is not None:+ data_payload = parsed_args.data_raw++ elif parsed_args.data is not None:+ data_payload = parsed_args.data++ elif parsed_args.data_urlencode:+ # Combine and parse urlencoded data+ combined_data = "&".join(parsed_args.data_urlencode)+ try:+ data_payload = dict(parse_qsl(combined_data, keep_blank_values=True))+ except Exception as e:+ log.warning(+ f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string."+ )+ data_payload = combined_data++ # Check if raw data looks like JSON, prefer 'json' param if so+ if isinstance(data_payload, str):+ try:+ maybe_json = json.loads(data_payload)+ if isinstance(maybe_json, (dict, list)):+ json_payload = maybe_json+ data_payload = None+ except json.JSONDecodeError:+ pass # Not JSON, keep it in data_payload++ # Handle -G: Move data to params if method is GET+ if method == "get" and data_payload:+ if isinstance(data_payload, dict): # From --data-urlencode likely+ params.update(data_payload)+ elif isinstance(data_payload, str):+ try:+ params.update(dict(parse_qsl(data_payload, keep_blank_values=True)))+ except ValueError:+ log.warning(+ f"Could not parse data '{data_payload}' into GET parameters for -G."+ )++ if params:+ data_payload = None # Clear data as it's moved to params+ json_payload = None # Should not have JSON body with -G++ # --- Process Proxy ---+ proxies: Optional[Dict[str, str]] = None+ if parsed_args.proxy:+ proxy_url = (+ f"http://{parsed_args.proxy}"+ if "://" not in parsed_args.proxy+ else parsed_args.proxy+ )++ if parsed_args.proxy_user:+ user_pass = parsed_args.proxy_user+ parts = urlparse(proxy_url)+ netloc_parts = parts.netloc.split("@")+ netloc = (+ f"{user_pass}@{netloc_parts[-1]}"+ if len(netloc_parts) > 1+ else f"{user_pass}@{parts.netloc}"+ )+ proxy_url = urlunparse(+ (+ parts.scheme,+ netloc,+ parts.path,+ parts.params,+ parts.query,+ parts.fragment,+ )+ )++ # Standard proxy dict format+ proxies = {"http": proxy_url, "https": proxy_url}+ log.debug(f"Using proxy configuration: {proxies}")++ # --- Final Context ---+ return Request(+ method=method,+ url=parsed_args.url,+ params=params,+ data=data_payload,+ json_data=json_payload,+ headers=headers,+ cookies=cookies,+ proxy=proxies,+ follow_redirects=True, # Scrapling default is True+ )++ def convert2fetcher(self, curl_command: [Request, str]) -> Optional[Response]:+ request = None+ if isinstance(curl_command, (Request, str)):+ request = (+ self.parse(curl_command)+ if isinstance(curl_command, str)+ else curl_command+ )+ request_args = request._asdict()+ method = request_args.pop("method").strip().lower()+ if method in self._supported_methods:+ request_args["json"] = request_args.pop("json_data")+ if method not in ("post", "put"):+ _ = request_args.pop("data")+ _ = request_args.pop("json")++ return getattr(Fetcher, method)(**request_args)+ else:+ log.error(+ f'Request method "{method}" isn\'t supported by Scrapling yet'+ )++ if request is None:+ log.error(+ "This class accepts `Request` objects only generated by the `uncurl` command or a curl command passed as string."+ )++ return None++def show_page_in_browser(page):if not page:log.error("Input must be of type `Adaptor`")return- fd, fname = tempfile.mkstemp(".html")+ fd, fname = make_temp_file(".html")os.write(fd, page.body.encode("utf-8"))os.close(fd)- webbrowser.open(f"file://{fname}")+ open_in_browser(f"file://{fname}")class CustomShell:@@ -40,13 +364,14 @@ class CustomShell:self.code = codeself.page = Noneself.pages = Adaptors([])+ self._curl_parser = CurlParser()log_level = log_level.strip().lower()if _known_logging_levels.get(log_level):self.log_level = _known_logging_levels[log_level]else:log.error(f'Unknown log level "{log_level}", defaulting to "DEBUG"')- self.log_level = logging.DEBUG+ self.log_level = DEBUGself.shell = None@@ -57,13 +382,13 @@ class CustomShell:"""Initialize application components"""# This is where you'd set up your application-specific objectsif self.log_level:- logging.getLogger("scrapling").setLevel(self.log_level)+ getLogger("scrapling").setLevel(self.log_level)settings = Fetcher.display_config()_ = settings.pop("storage")_ = settings.pop("storage_args")log.info(f"Scrapling {__version__} shell started")- log.info(f"Logging level is set to '{logging.getLevelName(self.log_level)}'")+ log.info(f"Logging level is set to '{getLevelName(self.log_level)}'")log.info(f"Fetchers' parsing settings: {settings}")@staticmethod@@ -87,6 +412,8 @@ class CustomShell:-> Useful commands- {"page / response":<30} The response object of the last page you fetched- {"pages":<30} Adaptors object of the last 5 response objects you fetched+ - {"uncurl('curl_command')":<30} Convert a curl command to a Fetcher's request and return the Request object for you. (Optimized to handle curl commands copied from DevTools network tab.)+ - {"curl2fetcher('curl_command')":<30} Convert a curl command to a Fetcher's request and execute it. (Optimized to handle curl commands copied from DevTools network tab.)- {"view(page)":<30} View page in a browser- {"help()":<30} Show this help message (Shell help)@@ -146,6 +473,8 @@ Type 'exit' or press Ctrl+D to exit."response": self.page,"pages": self.pages,"view": show_page_in_browser,+ "uncurl": self._curl_parser.parse,+ "curl2fetcher": self._curl_parser.convert2fetcher,"help": self.show_help,}