🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor: Make all fetchers as an optional dependency group
Karim shoair13d7e70cb70e
diff --git a/pyproject.toml b/pyproject.tomlindex a1aa1af..5c708f3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -61,6 +61,10 @@ dependencies = ["click>=8.2.1","orjson>=3.11.3","tldextract>=5.3.0",+]++[project.optional-dependencies]+fetchers = ["curl_cffi>=0.13.0","playwright>=1.52.0","rebrowser-playwright>=1.52.0",@@ -68,15 +72,15 @@ dependencies = ["geoip2>=5.1.0","msgspec>=0.19.0",]--[project.optional-dependencies]ai = ["mcp>=1.14.0","markdownify>=1.2.0",+ "scrapling[fetchers]",]shell = ["IPython>=8.37", # The last version that supports Python 3.10"markdownify>=1.2.0",+ "scrapling[fetchers]",]all = ["scrapling[ai,shell]",diff --git a/scrapling/cli.py b/scrapling/cli.pyindex e4b3c7f..3e052ac 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -2,11 +2,9 @@ from pathlib import Pathfrom subprocess import check_outputfrom sys import executable as python_executable-from scrapling.core.utils import log-from scrapling.engines.toolbelt import Response+from scrapling.engines.toolbelt.custom import Response+from scrapling.core.utils import log, _CookieParser, _ParseHeadersfrom scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable-from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher-from scrapling.core.shell import Convertor, _CookieParser, _ParseHeadersfrom orjson import loads as json_loads, JSONDecodeErrorfrom click import command, option, Choice, group, argument@@ -40,6 +38,8 @@ def __Request_and_Save(**kwargs,) -> None:"""Make a request using the specified fetcher function and save the result"""+ from scrapling.core.shell import Convertor+# Handle relative paths - convert to an absolute path based on the current working directoryoutput_path = Path(output_file)if not output_path.is_absolute():@@ -251,6 +251,8 @@ def get(impersonate=impersonate,proxy=proxy,)+ from scrapling.fetchers import Fetcher+__Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs)@@ -347,6 +349,8 @@ def post(proxy=proxy,data=data,)+ from scrapling.fetchers import Fetcher+__Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs)@@ -439,6 +443,8 @@ def put(proxy=proxy,data=data,)+ from scrapling.fetchers import Fetcher+__Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs)@@ -524,6 +530,8 @@ def delete(impersonate=impersonate,proxy=proxy,)+ from scrapling.fetchers import Fetcher+__Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs)@@ -643,6 +651,8 @@ def fetch(if parsed_headers:kwargs["extra_headers"] = parsed_headers+ from scrapling.fetchers import DynamicFetcher+__Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs)@@ -790,6 +800,8 @@ def stealthy_fetch(if parsed_headers:kwargs["extra_headers"] = parsed_headers+ from scrapling.fetchers import StealthyFetcher+__Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs)diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 8459a60..2b23b2b 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -2,7 +2,6 @@from re import sub as re_subfrom sys import stderrfrom functools import wraps-from http import cookies as Cookiefrom collections import namedtuplefrom shlex import split as shlex_splitfrom tempfile import mkstemp as make_temp_file@@ -23,25 +22,17 @@ from logging import (from orjson import loads as json_loads, JSONDecodeErrorfrom scrapling import __version__-from scrapling.core.custom_types import TextHandler-from scrapling.core.utils import logfrom scrapling.parser import Selector, Selectors+from scrapling.core.custom_types import TextHandler+from scrapling.engines.toolbelt.custom import Response+from scrapling.core.utils import log, _ParseHeaders, _CookieParserfrom scrapling.core._types import (- List,Optional,Dict,- Tuple,Any,extraction_types,Generator,)-from scrapling.fetchers import (- Fetcher,- AsyncFetcher,- DynamicFetcher,- StealthyFetcher,- Response,-)_known_logging_levels = {@@ -71,46 +62,6 @@ Request = namedtuple()-def _CookieParser(cookie_string):- # Errors will be handled on call so the log can be specified- cookie_parser = Cookie.SimpleCookie()- cookie_parser.load(cookie_string)- for key, morsel in cookie_parser.items():- yield key, morsel.value---def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:- """Parses headers into separate header and cookie dictionaries."""- header_dict = dict()- cookie_dict = dict()-- for header_line in header_lines:- if ":" not in header_line:- if header_line.endswith(";"):- header_key = header_line[:-1].strip()- header_value = ""- header_dict[header_key] = header_value- else:- raise ValueError(f"Could not parse header without colon: '{header_line}'.")- else:- header_key, header_value = header_line.split(":", 1)- header_key = header_key.strip()- header_value = header_value.strip()-- if parse_cookies:- if header_key.lower() == "cookie":- try:- cookie_dict = {key: value for key, value in _CookieParser(header_value)}- except Exception as e: # pragma: no cover- raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")- else:- header_dict[header_key] = header_value- else:- header_dict[header_key] = header_value-- return header_dict, cookie_dict--# Suppress exit on error to handle parsing errors gracefullyclass NoExitArgumentParser(ArgumentParser): # pragma: no coverdef error(self, message):@@ -128,6 +79,9 @@ class CurlParser:"""Builds the argument parser for relevant curl flags from DevTools."""def __init__(self):+ from scrapling.fetchers import Fetcher as __Fetcher++ self.__fetcher = __Fetcher# We will use argparse parser to parse the curl command directly instead of regex# We will focus more on flags that will show up on curl commands copied from DevTools's network tab_parser = NoExitArgumentParser(add_help=False) # Disable default help@@ -343,7 +297,7 @@ class CurlParser:_ = request_args.pop("json", None)try:- return getattr(Fetcher, method)(**request_args)+ return getattr(self.__Fetcher, method)(**request_args)except Exception as e: # pragma: no coverlog.error(f"Error calling Fetcher.{method}: {e}")return None@@ -377,6 +331,19 @@ class CustomShell:"""A custom IPython shell with minimal dependencies"""def __init__(self, code, log_level="debug"):+ from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed+ from scrapling.fetchers import (+ Fetcher as __Fetcher,+ AsyncFetcher as __AsyncFetcher,+ DynamicFetcher as __DynamicFetcher,+ StealthyFetcher as __StealthyFetcher,+ )++ self.__InteractiveShellEmbed = __InteractiveShellEmbed+ self.__Fetcher = __Fetcher+ self.__AsyncFetcher = __AsyncFetcher+ self.__DynamicFetcher = __DynamicFetcher+ self.__StealthyFetcher = __StealthyFetcherself.code = codeself.page = Noneself.pages = Selectors([])@@ -400,7 +367,7 @@ class CustomShell:if self.log_level:getLogger("scrapling").setLevel(self.log_level)- settings = Fetcher.display_config()+ settings = self.__Fetcher.display_config()settings.pop("storage", None)settings.pop("storage_args", None)log.info(f"Scrapling {__version__} shell started")@@ -466,12 +433,12 @@ Type 'exit' or press Ctrl+D to exit."""Create a namespace with application-specific objects"""# Create wrapped versions of fetch functions- get = self.create_wrapper(Fetcher.get)- post = self.create_wrapper(Fetcher.post)- put = self.create_wrapper(Fetcher.put)- delete = self.create_wrapper(Fetcher.delete)- dynamic_fetch = self.create_wrapper(DynamicFetcher.fetch)- stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)+ get = self.create_wrapper(self.__Fetcher.get)+ post = self.create_wrapper(self.__Fetcher.post)+ put = self.create_wrapper(self.__Fetcher.put)+ delete = self.create_wrapper(self.__Fetcher.delete)+ dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)+ stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch)curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher)# Create the namespace dictionary@@ -480,12 +447,12 @@ Type 'exit' or press Ctrl+D to exit."post": post,"put": put,"delete": delete,- "Fetcher": Fetcher,- "AsyncFetcher": AsyncFetcher,+ "Fetcher": self.__Fetcher,+ "AsyncFetcher": self.__AsyncFetcher,"fetch": dynamic_fetch,- "DynamicFetcher": DynamicFetcher,+ "DynamicFetcher": self.__DynamicFetcher,"stealthy_fetch": stealthy_fetch,- "StealthyFetcher": StealthyFetcher,+ "StealthyFetcher": self.__StealthyFetcher,"Selector": Selector,"page": self.page,"response": self.page,@@ -502,11 +469,10 @@ Type 'exit' or press Ctrl+D to exit.def start(self): # pragma: no cover"""Start the interactive shell"""- from IPython.terminal.embed import InteractiveShellEmbed# Get our namespace with application objectsnamespace = self.get_namespace()- ipython_shell = InteractiveShellEmbed(+ ipython_shell = self.__InteractiveShellEmbed(banner1=self.banner(),banner2="",enable_tip=False,diff --git a/scrapling/core/translator.py b/scrapling/core/translator.pyindex 88dfbff..d98092e 100644--- a/scrapling/core/translator.py+++ b/scrapling/core/translator.py@@ -10,10 +10,10 @@ So you don't have to learn a new selectors/api method like what bs4 done with sofrom functools import lru_cache-from cssselect import HTMLTranslator as OriginalHTMLTranslator-from cssselect.parser import Element, FunctionalPseudoElement, PseudoElementfrom cssselect.xpath import ExpressionErrorfrom cssselect.xpath import XPathExpr as OriginalXPathExpr+from cssselect import HTMLTranslator as OriginalHTMLTranslator+from cssselect.parser import Element, FunctionalPseudoElement, PseudoElementfrom scrapling.core._types import Any, Optional, Protocol, Selfdiff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.pynew file mode 100644index 0000000..dc95705--- /dev/null+++ b/scrapling/core/utils/__init__.py@@ -0,0 +1,10 @@+from ._utils import (+ log,+ __CONSECUTIVE_SPACES_REGEX__,+ flatten,+ _is_iterable,+ _StorageTools,+ clean_spaces,+ html_forbidden,+)+from ._shell import _CookieParser, _ParseHeadersdiff --git a/scrapling/core/utils/_shell.py b/scrapling/core/utils/_shell.pynew file mode 100644index 0000000..05420ae--- /dev/null+++ b/scrapling/core/utils/_shell.py@@ -0,0 +1,48 @@+from http import cookies as Cookie+++from scrapling.core._types import (+ List,+ Dict,+ Tuple,+)+++def _CookieParser(cookie_string):+ # Errors will be handled on call so the log can be specified+ cookie_parser = Cookie.SimpleCookie()+ cookie_parser.load(cookie_string)+ for key, morsel in cookie_parser.items():+ yield key, morsel.value+++def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:+ """Parses headers into separate header and cookie dictionaries."""+ header_dict = dict()+ cookie_dict = dict()++ for header_line in header_lines:+ if ":" not in header_line:+ if header_line.endswith(";"):+ header_key = header_line[:-1].strip()+ header_value = ""+ header_dict[header_key] = header_value+ else:+ raise ValueError(f"Could not parse header without colon: '{header_line}'.")+ else:+ header_key, header_value = header_line.split(":", 1)+ header_key = header_key.strip()+ header_value = header_value.strip()++ if parse_cookies:+ if header_key.lower() == "cookie":+ try:+ cookie_dict = {key: value for key, value in _CookieParser(header_value)}+ except Exception as e: # pragma: no cover+ raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")+ else:+ header_dict[header_key] = header_value+ else:+ header_dict[header_key] = header_value++ return header_dict, cookie_dictdiff --git a/scrapling/core/utils.py b/scrapling/core/utils/_utils.pysimilarity index 100%rename from scrapling/core/utils.pyrename to scrapling/core/utils/_utils.pydiff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.pyindex 477bb21..e69de29 100644--- a/scrapling/engines/__init__.py+++ b/scrapling/engines/__init__.py@@ -1,16 +0,0 @@-from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS-from .static import FetcherSession, FetcherClient, AsyncFetcherClient-from ._browsers import (- DynamicSession,- AsyncDynamicSession,- StealthySession,- AsyncStealthySession,-)--__all__ = [- "FetcherSession",- "DynamicSession",- "AsyncDynamicSession",- "StealthySession",- "AsyncStealthySession",-]diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 5bb86ab..00c5567 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -12,20 +12,17 @@ from camoufox.utils import (installed_verstr as camoufox_version,)-from scrapling.engines.toolbelt import (- intercept_route,- async_intercept_route,- get_os_name,-)-from ._page import PageInfo, PagePool-from ._config_tools import _compiled_stealth_scripts-from ._validators import validate, PlaywrightConfig, CamoufoxConfig-from ._config_tools import _launch_kwargs, _context_kwargs+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_routefrom scrapling.core._types import (Any,Dict,Optional,)+from ._page import PageInfo, PagePool+from ._config_tools import _compiled_stealth_scripts+from ._config_tools import _launch_kwargs, _context_kwargs+from scrapling.engines.toolbelt.fingerprints import get_os_name+from ._validators import validate, PlaywrightConfig, CamoufoxConfig__ff_version_str__ = camoufox_version().split(".", 1)[0]diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 31113be..6e0bc1e 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -25,11 +25,11 @@ from scrapling.core._types import (Callable,SelectorWaitStates,)-from scrapling.engines.toolbelt import (+from scrapling.engines.toolbelt.convertor import (Response,ResponseFactory,- generate_convincing_referer,)+from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")_UNSET = object()diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.pyindex 96b7f41..4b405e6 100644--- a/scrapling/engines/_browsers/_config_tools.py+++ b/scrapling/engines/_browsers/_config_tools.py@@ -6,7 +6,8 @@ from scrapling.engines.constants import (HARMFUL_DEFAULT_ARGS,DEFAULT_FLAGS,)-from scrapling.engines.toolbelt import js_bypass_path, generate_headers+from scrapling.engines.toolbelt.navigation import js_bypass_path+from scrapling.engines.toolbelt.fingerprints import generate_headers__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 96d7291..07efee2 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -26,11 +26,11 @@ from scrapling.core._types import (Callable,SelectorWaitStates,)-from scrapling.engines.toolbelt import (+from scrapling.engines.toolbelt.convertor import (Response,ResponseFactory,- generate_convincing_referer,)+from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer_UNSET = object()diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex a20ae55..ca64942 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -9,7 +9,7 @@ from scrapling.core._types import (List,SelectorWaitStates,)-from scrapling.engines.toolbelt import construct_proxy_dict+from scrapling.engines.toolbelt.navigation import construct_proxy_dictclass PlaywrightConfig(Struct, kw_only=True, frozen=False):diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex e028feb..35271b8 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -26,11 +26,11 @@ from scrapling.core._types import (from .toolbelt import (Response,- generate_convincing_referer,generate_headers,- ResponseFactory,__default_useragent__,)+from .toolbelt.convertor import ResponseFactory+from .toolbelt.fingerprints import generate_convincing_referer_UNSET = object()diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex d58fd57..04f43b1 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -5,16 +5,7 @@ from .custom import (get_variable_name,)from .fingerprints import (- generate_convincing_referer,generate_headers,get_os_name,__default_useragent__,)-from .navigation import (- async_intercept_route,- construct_cdp_url,- construct_proxy_dict,- intercept_route,- js_bypass_path,-)-from .convertor import ResponseFactorydiff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 72c39e9..f9effa2 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -2,8 +2,10 @@Functions related to custom types or type checking"""+from functools import lru_cachefrom email.message import Message+from scrapling.core.utils import logfrom scrapling.core._types import (Any,Dict,@@ -12,7 +14,6 @@ from scrapling.core._types import (Tuple,)from scrapling.core.custom_types import MappingProxyType-from scrapling.core.utils import log, lru_cachefrom scrapling.parser import Selector, SQLiteStorageSystemdiff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 4ca8e38..bd836e7 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -2,13 +2,13 @@Functions related to generating headers and fingerprints generally"""+from functools import lru_cachefrom platform import system as platform_systemfrom tldextract import extractfrom browserforge.headers import Browser, HeaderGeneratorfrom scrapling.core._types import Dict, Optional-from scrapling.core.utils import lru_cache__OS_NAME__ = platform_system()@@ -37,8 +37,6 @@ def get_os_name() -> Optional[str]:"Linux": "linux","Darwin": "macos","Windows": "windows",- # For the future? because why not?- "iOS": "ios",}.get(__OS_NAME__)diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 6d666a8..f2f445c 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -86,51 +86,6 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) ->return None-def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:- """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists-- :param cdp_url: The target URL.- :param query_params: A dictionary of the parameters to add.- :return: The new CDP URL.- """- try:- # Validate the base URL structure- parsed = urlparse(cdp_url)-- # Check scheme- if parsed.scheme not in ("ws", "wss"):- raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")-- # Validate hostname and port- if not parsed.netloc:- raise ValueError("Invalid hostname for the CDP URL")-- try:- # Checking if the port is valid (if available)- _ = parsed.port- except ValueError:- # urlparse will raise `ValueError` if the port can't be casted to integer- raise ValueError("Invalid port for the CDP URL")-- # Ensure the path starts with /- path = parsed.path- if not path.startswith("/"):- path = "/" + path-- # Reconstruct the base URL with validated parts- validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"-- # Add query parameters- if query_params:- query_string = urlencode(query_params)- return f"{validated_base}?{query_string}"-- return validated_base-- except Exception as e:- raise ValueError(f"Invalid CDP URL: {str(e)}")--@lru_cache(10, typed=True)def js_bypass_path(filename: str) -> str:"""Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of itdiff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 2421b42..323293d 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -6,15 +6,17 @@ from scrapling.core._types import (SelectorWaitStates,Iterable,)-from scrapling.engines import (+from scrapling.engines.static import (FetcherSession,- StealthySession,- AsyncStealthySession,- DynamicSession,- AsyncDynamicSession,FetcherClient as _FetcherClient,AsyncFetcherClient as _AsyncFetcherClient,)+from scrapling.engines._browsers import (+ DynamicSession,+ StealthySession,+ AsyncDynamicSession,+ AsyncStealthySession,+)from scrapling.engines.toolbelt import BaseFetcher, Response__FetcherClientInstance__ = _FetcherClient()diff --git a/scrapling/parser.py b/scrapling/parser.pyindex f943db8..5335a82 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -1,12 +1,11 @@-from pathlib import Pathimport re+from pathlib import Pathfrom inspect import signature-from difflib import SequenceMatcherfrom urllib.parse import urljoin+from difflib import SequenceMatcher-from cssselect import SelectorError, SelectorSyntaxError-from cssselect import parse as split_selectorsfrom lxml.html import HtmlElement, HtmlMixin, HTMLParser+from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectorsfrom lxml.etree import (XPath,tostring,