Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: update shell signatures for the autocompletion

Karim shoair842718ba94de
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index 803e6d5..d9a42fc 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -1,9 +1,13 @@
from scrapling.core._types import (
- Dict,
Any,
+ Dict,
List,
Tuple,
+ Sequence,
+ Callable,
Optional,
+ SetCookieParam,
+ SelectorWaitStates,
)
# Parameter definitions for shell function signatures (defined once at module level)
@@ -31,57 +35,58 @@ _REQUESTS_PARAMS = {
_FETCH_PARAMS = {
"headless": bool,
+ "disable_resources": bool,
+ "network_idle": bool,
+ "load_dom": bool,
+ "wait_selector": Optional[str],
+ "wait_selector_state": SelectorWaitStates,
+ "cookies": Sequence[SetCookieParam],
"google_search": bool,
- "hide_canvas": bool,
- "disable_webgl": bool,
- "real_chrome": bool,
- "stealth": bool,
"wait": int | float,
- "page_action": Optional[Any],
- "proxy": Optional[str | Dict],
- "locale": str,
+ "timezone_id": str | None,
+ "page_action": Optional[Callable],
+ "proxy": Optional[str | Dict[str, str] | Tuple],
"extra_headers": Optional[Dict[str, str]],
- "useragent": Optional[str],
- "cdp_url": Optional[str],
"timeout": int | float,
- "disable_resources": bool,
- "wait_selector": Optional[str],
"init_script": Optional[str],
- "cookies": Optional[List[Dict]],
- "network_idle": bool,
- "load_dom": bool,
- "wait_selector_state": Any,
- "extra_flags": Optional[List[str]],
- "additional_args": Optional[Dict],
+ "user_data_dir": str,
"selector_config": Optional[Dict],
+ "additional_args": Optional[Dict],
+ "locale": Optional[str],
+ "real_chrome": bool,
+ "cdp_url": Optional[str],
+ "useragent": Optional[str],
+ "extra_flags": Optional[List[str]],
}
_STEALTHY_FETCH_PARAMS = {
"headless": bool,
- "block_images": bool,
"disable_resources": bool,
- "block_webrtc": bool,
- "allow_webgl": bool,
"network_idle": bool,
"load_dom": bool,
- "humanize": bool | float,
- "solve_cloudflare": bool,
- "wait": int | float,
- "timeout": int | float,
- "page_action": Optional[Any],
"wait_selector": Optional[str],
- "init_script": Optional[str],
- "addons": Optional[List[str]],
- "wait_selector_state": Any,
- "cookies": Optional[List[Dict]],
+ "wait_selector_state": SelectorWaitStates,
+ "cookies": Sequence[SetCookieParam],
"google_search": bool,
+ "wait": int | float,
+ "timezone_id": str | None,
+ "page_action": Optional[Callable],
+ "proxy": Optional[str | Dict[str, str] | Tuple],
"extra_headers": Optional[Dict[str, str]],
- "proxy": Optional[str | Dict],
- "os_randomize": bool,
- "disable_ads": bool,
- "geoip": bool,
+ "timeout": int | float,
+ "init_script": Optional[str],
+ "user_data_dir": str,
"selector_config": Optional[Dict],
"additional_args": Optional[Dict],
+ "locale": Optional[str],
+ "real_chrome": bool,
+ "cdp_url": Optional[str],
+ "useragent": Optional[str],
+ "extra_flags": Optional[List[str]],
+ "allow_webgl": bool,
+ "hide_canvas": bool,
+ "block_webrtc": bool,
+ "solve_cloudflare": bool,
}
# Mapping of function names to their parameter definitions
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 1cdbb9a..07cb90a 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -11,9 +11,10 @@ from scrapling.core._types import (
Mapping,
Optional,
Callable,
- Iterable,
+ Sequence,
TypedDict,
TypeAlias,
+ SetCookieParam,
SelectorWaitStates,
TYPE_CHECKING,
)
@@ -61,7 +62,7 @@ if TYPE_CHECKING: # pragma: no cover
load_dom: bool
wait_selector: Optional[str]
wait_selector_state: SelectorWaitStates
- cookies: Optional[Iterable[Dict]]
+ cookies: Sequence[SetCookieParam]
google_search: bool
wait: int | float
timezone_id: str | None
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index cb3f48a..b148fe6 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -67,32 +67,29 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
"""Configuration struct for validation"""
max_pages: PagesCount = 1
- cdp_url: Optional[str] = None
headless: bool = True
+ disable_resources: bool = False
+ network_idle: bool = False
+ load_dom: bool = True
+ wait_selector: Optional[str] = None
+ wait_selector_state: SelectorWaitStates = "attached"
+ cookies: Sequence[SetCookieParam] | None = []
google_search: bool = True
- # hide_canvas: bool = False
- # disable_webgl: bool = False
- real_chrome: bool = False
- # stealth: bool = False
wait: Seconds = 0
+ timezone_id: str | None = ""
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
- locale: str | None = None
extra_headers: Optional[Dict[str, str]] = None
- useragent: Optional[str] = None
timeout: Seconds = 30000
init_script: Optional[str] = None
- disable_resources: bool = False
- wait_selector: Optional[str] = None
- cookies: Sequence[SetCookieParam] | None = []
- network_idle: bool = False
- load_dom: bool = True
- wait_selector_state: SelectorWaitStates = "attached"
user_data_dir: str = ""
- timezone_id: str | None = ""
- extra_flags: Optional[List[str]] = None
selector_config: Optional[Dict] = {}
additional_args: Optional[Dict] = {}
+ locale: str | None = None
+ real_chrome: bool = False
+ cdp_url: Optional[str] = None
+ useragent: Optional[str] = None
+ extra_flags: Optional[List[str]] = None
def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation"""