Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: make browser fetchers type hints always present

Karim shoairba4d33337f4a
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index cd5ea19..30d6af1 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -19,7 +19,6 @@ from scrapling.core._types import (
TypeAlias,
SetCookieParam,
SelectorWaitStates,
- TYPE_CHECKING,
)
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
@@ -27,97 +26,93 @@ from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
ImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
-if TYPE_CHECKING: # pragma: no cover
- # Types for session initialization
- class RequestsSession(TypedDict, total=False):
- impersonate: ImpersonateType
- http3: Optional[bool]
- stealthy_headers: Optional[bool]
- proxies: Optional[ProxySpec]
- proxy: Optional[str]
- proxy_auth: Optional[Tuple[str, str]]
- proxy_rotator: Optional[ProxyRotator]
- timeout: Optional[int | float]
- headers: Optional[Mapping[str, Optional[str]]]
- retries: Optional[int]
- retry_delay: Optional[int]
- follow_redirects: Optional[bool]
- max_redirects: Optional[int]
- verify: Optional[bool]
- cert: Optional[str | Tuple[str, str]]
- selector_config: Optional[Dict]
-
- # Types for GET request method parameters
- class GetRequestParams(RequestsSession, total=False):
- params: Optional[Dict | List | Tuple]
- cookies: Optional[CookieTypes]
- auth: Optional[Tuple[str, str]]
-
- # Types for POST/PUT/DELETE request method parameters
- class DataRequestParams(GetRequestParams, total=False):
- data: Optional[Dict[str, str] | List[Tuple] | str | BytesIO | bytes]
- json: Optional[Dict | List]
-
- # Types for browser session
- class PlaywrightSession(TypedDict, total=False):
- max_pages: int
- headless: bool
- disable_resources: bool
- network_idle: bool
- load_dom: bool
- wait_selector: Optional[str]
- wait_selector_state: SelectorWaitStates
- cookies: Sequence[SetCookieParam] | None
- google_search: bool
- wait: int | float
- timezone_id: str | None
- page_action: Optional[Callable]
- proxy: Optional[str | Dict[str, str] | Tuple]
- proxy_rotator: Optional[ProxyRotator]
- extra_headers: Optional[Dict[str, str]]
- timeout: int | float
- init_script: Optional[str]
- user_data_dir: str
- selector_config: Optional[Dict]
- additional_args: Optional[Dict]
- locale: Optional[str]
- real_chrome: bool
- cdp_url: Optional[str]
- useragent: Optional[str]
- extra_flags: Optional[List[str]]
- blocked_domains: Optional[Set[str]]
- retries: int
- retry_delay: int | float
-
- class PlaywrightFetchParams(TypedDict, total=False):
- load_dom: bool
- wait: int | float
- network_idle: bool
- google_search: bool
- timeout: int | float
- disable_resources: bool
- wait_selector: Optional[str]
- page_action: Optional[Callable]
- selector_config: Optional[Dict]
- extra_headers: Optional[Dict[str, str]]
- wait_selector_state: SelectorWaitStates
- blocked_domains: Optional[Set[str]]
- proxy: Optional[str | Dict[str, str]]
-
- class StealthSession(PlaywrightSession, total=False):
- allow_webgl: bool
- hide_canvas: bool
- block_webrtc: bool
- solve_cloudflare: bool
-
- class StealthFetchParams(PlaywrightFetchParams, total=False):
- solve_cloudflare: bool
-
-else: # pragma: no cover
- RequestsSession = TypedDict
- GetRequestParams = TypedDict
- DataRequestParams = TypedDict
- PlaywrightSession = TypedDict
- PlaywrightFetchParams = TypedDict
- StealthSession = TypedDict
- StealthFetchParams = TypedDict
+# Types for session initialization
+class RequestsSession(TypedDict, total=False):
+ impersonate: ImpersonateType
+ http3: Optional[bool]
+ stealthy_headers: Optional[bool]
+ proxies: Optional[ProxySpec]
+ proxy: Optional[str]
+ proxy_auth: Optional[Tuple[str, str]]
+ proxy_rotator: Optional[ProxyRotator]
+ timeout: Optional[int | float]
+ headers: Optional[Mapping[str, Optional[str]]]
+ retries: Optional[int]
+ retry_delay: Optional[int]
+ follow_redirects: Optional[bool]
+ max_redirects: Optional[int]
+ verify: Optional[bool]
+ cert: Optional[str | Tuple[str, str]]
+ selector_config: Optional[Dict]
+
+
+# Types for GET request method parameters
+class GetRequestParams(RequestsSession, total=False):
+ params: Optional[Dict | List | Tuple]
+ cookies: Optional[CookieTypes]
+ auth: Optional[Tuple[str, str]]
+
+
+# Types for POST/PUT/DELETE request method parameters
+class DataRequestParams(GetRequestParams, total=False):
+ data: Optional[Dict[str, str] | List[Tuple] | str | BytesIO | bytes]
+ json: Optional[Dict | List]
+
+
+# Types for browser session
+class PlaywrightSession(TypedDict, total=False):
+ max_pages: int
+ headless: bool
+ disable_resources: bool
+ network_idle: bool
+ load_dom: bool
+ wait_selector: Optional[str]
+ wait_selector_state: SelectorWaitStates
+ cookies: Sequence[SetCookieParam] | None
+ google_search: bool
+ wait: int | float
+ timezone_id: str | None
+ page_action: Optional[Callable]
+ proxy: Optional[str | Dict[str, str] | Tuple]
+ proxy_rotator: Optional[ProxyRotator]
+ extra_headers: Optional[Dict[str, str]]
+ timeout: int | float
+ init_script: Optional[str]
+ user_data_dir: str
+ selector_config: Optional[Dict]
+ additional_args: Optional[Dict]
+ locale: Optional[str]
+ real_chrome: bool
+ cdp_url: Optional[str]
+ useragent: Optional[str]
+ extra_flags: Optional[List[str]]
+ blocked_domains: Optional[Set[str]]
+ retries: int
+ retry_delay: int | float
+
+
+class PlaywrightFetchParams(TypedDict, total=False):
+ load_dom: bool
+ wait: int | float
+ network_idle: bool
+ google_search: bool
+ timeout: int | float
+ disable_resources: bool
+ wait_selector: Optional[str]
+ page_action: Optional[Callable]
+ selector_config: Optional[Dict]
+ extra_headers: Optional[Dict[str, str]]
+ wait_selector_state: SelectorWaitStates
+ blocked_domains: Optional[Set[str]]
+ proxy: Optional[str | Dict[str, str]]
+
+
+class StealthSession(PlaywrightSession, total=False):
+ allow_webgl: bool
+ hide_canvas: bool
+ block_webrtc: bool
+ solve_cloudflare: bool
+
+
+class StealthFetchParams(PlaywrightFetchParams, total=False):
+ solve_cloudflare: bool