🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: update shell signatures for the autocompletion
Karim shoair842718ba94de
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex 803e6d5..d9a42fc 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -1,9 +1,13 @@from scrapling.core._types import (- Dict,Any,+ Dict,List,Tuple,+ Sequence,+ Callable,Optional,+ SetCookieParam,+ SelectorWaitStates,)# Parameter definitions for shell function signatures (defined once at module level)@@ -31,57 +35,58 @@ _REQUESTS_PARAMS = {_FETCH_PARAMS = {"headless": bool,+ "disable_resources": bool,+ "network_idle": bool,+ "load_dom": bool,+ "wait_selector": Optional[str],+ "wait_selector_state": SelectorWaitStates,+ "cookies": Sequence[SetCookieParam],"google_search": bool,- "hide_canvas": bool,- "disable_webgl": bool,- "real_chrome": bool,- "stealth": bool,"wait": int | float,- "page_action": Optional[Any],- "proxy": Optional[str | Dict],- "locale": str,+ "timezone_id": str | None,+ "page_action": Optional[Callable],+ "proxy": Optional[str | Dict[str, str] | Tuple],"extra_headers": Optional[Dict[str, str]],- "useragent": Optional[str],- "cdp_url": Optional[str],"timeout": int | float,- "disable_resources": bool,- "wait_selector": Optional[str],"init_script": Optional[str],- "cookies": Optional[List[Dict]],- "network_idle": bool,- "load_dom": bool,- "wait_selector_state": Any,- "extra_flags": Optional[List[str]],- "additional_args": Optional[Dict],+ "user_data_dir": str,"selector_config": Optional[Dict],+ "additional_args": Optional[Dict],+ "locale": Optional[str],+ "real_chrome": bool,+ "cdp_url": Optional[str],+ "useragent": Optional[str],+ "extra_flags": Optional[List[str]],}_STEALTHY_FETCH_PARAMS = {"headless": bool,- "block_images": bool,"disable_resources": bool,- "block_webrtc": bool,- "allow_webgl": bool,"network_idle": bool,"load_dom": bool,- "humanize": bool | float,- "solve_cloudflare": bool,- "wait": int | float,- "timeout": int | float,- "page_action": Optional[Any],"wait_selector": Optional[str],- "init_script": Optional[str],- "addons": Optional[List[str]],- "wait_selector_state": Any,- "cookies": Optional[List[Dict]],+ "wait_selector_state": SelectorWaitStates,+ "cookies": Sequence[SetCookieParam],"google_search": bool,+ "wait": int | float,+ "timezone_id": str | None,+ "page_action": Optional[Callable],+ "proxy": Optional[str | Dict[str, str] | Tuple],"extra_headers": Optional[Dict[str, str]],- "proxy": Optional[str | Dict],- "os_randomize": bool,- "disable_ads": bool,- "geoip": bool,+ "timeout": int | float,+ "init_script": Optional[str],+ "user_data_dir": str,"selector_config": Optional[Dict],"additional_args": Optional[Dict],+ "locale": Optional[str],+ "real_chrome": bool,+ "cdp_url": Optional[str],+ "useragent": Optional[str],+ "extra_flags": Optional[List[str]],+ "allow_webgl": bool,+ "hide_canvas": bool,+ "block_webrtc": bool,+ "solve_cloudflare": bool,}# Mapping of function names to their parameter definitionsdiff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 1cdbb9a..07cb90a 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -11,9 +11,10 @@ from scrapling.core._types import (Mapping,Optional,Callable,- Iterable,+ Sequence,TypedDict,TypeAlias,+ SetCookieParam,SelectorWaitStates,TYPE_CHECKING,)@@ -61,7 +62,7 @@ if TYPE_CHECKING: # pragma: no coverload_dom: boolwait_selector: Optional[str]wait_selector_state: SelectorWaitStates- cookies: Optional[Iterable[Dict]]+ cookies: Sequence[SetCookieParam]google_search: boolwait: int | floattimezone_id: str | Nonediff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex cb3f48a..b148fe6 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -67,32 +67,29 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):"""Configuration struct for validation"""max_pages: PagesCount = 1- cdp_url: Optional[str] = Noneheadless: bool = True+ disable_resources: bool = False+ network_idle: bool = False+ load_dom: bool = True+ wait_selector: Optional[str] = None+ wait_selector_state: SelectorWaitStates = "attached"+ cookies: Sequence[SetCookieParam] | None = []google_search: bool = True- # hide_canvas: bool = False- # disable_webgl: bool = False- real_chrome: bool = False- # stealth: bool = Falsewait: Seconds = 0+ timezone_id: str | None = ""page_action: Optional[Callable] = Noneproxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`- locale: str | None = Noneextra_headers: Optional[Dict[str, str]] = None- useragent: Optional[str] = Nonetimeout: Seconds = 30000init_script: Optional[str] = None- disable_resources: bool = False- wait_selector: Optional[str] = None- cookies: Sequence[SetCookieParam] | None = []- network_idle: bool = False- load_dom: bool = True- wait_selector_state: SelectorWaitStates = "attached"user_data_dir: str = ""- timezone_id: str | None = ""- extra_flags: Optional[List[str]] = Noneselector_config: Optional[Dict] = {}additional_args: Optional[Dict] = {}+ locale: str | None = None+ real_chrome: bool = False+ cdp_url: Optional[str] = None+ useragent: Optional[str] = None+ extra_flags: Optional[List[str]] = Nonedef __post_init__(self): # pragma: no cover"""Custom validation after msgspec validation"""