🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: make browser fetchers type hints always present
Karim shoairba4d33337f4a
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex cd5ea19..30d6af1 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -19,7 +19,6 @@ from scrapling.core._types import (TypeAlias,SetCookieParam,SelectorWaitStates,- TYPE_CHECKING,)from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator@@ -27,97 +26,93 @@ from scrapling.engines.toolbelt.proxy_rotation import ProxyRotatorImpersonateType: TypeAlias = BrowserTypeLiteral | List[BrowserTypeLiteral] | None-if TYPE_CHECKING: # pragma: no cover- # Types for session initialization- class RequestsSession(TypedDict, total=False):- impersonate: ImpersonateType- http3: Optional[bool]- stealthy_headers: Optional[bool]- proxies: Optional[ProxySpec]- proxy: Optional[str]- proxy_auth: Optional[Tuple[str, str]]- proxy_rotator: Optional[ProxyRotator]- timeout: Optional[int | float]- headers: Optional[Mapping[str, Optional[str]]]- retries: Optional[int]- retry_delay: Optional[int]- follow_redirects: Optional[bool]- max_redirects: Optional[int]- verify: Optional[bool]- cert: Optional[str | Tuple[str, str]]- selector_config: Optional[Dict]-- # Types for GET request method parameters- class GetRequestParams(RequestsSession, total=False):- params: Optional[Dict | List | Tuple]- cookies: Optional[CookieTypes]- auth: Optional[Tuple[str, str]]-- # Types for POST/PUT/DELETE request method parameters- class DataRequestParams(GetRequestParams, total=False):- data: Optional[Dict[str, str] | List[Tuple] | str | BytesIO | bytes]- json: Optional[Dict | List]-- # Types for browser session- class PlaywrightSession(TypedDict, total=False):- max_pages: int- headless: bool- disable_resources: bool- network_idle: bool- load_dom: bool- wait_selector: Optional[str]- wait_selector_state: SelectorWaitStates- cookies: Sequence[SetCookieParam] | None- google_search: bool- wait: int | float- timezone_id: str | None- page_action: Optional[Callable]- proxy: Optional[str | Dict[str, str] | Tuple]- proxy_rotator: Optional[ProxyRotator]- extra_headers: Optional[Dict[str, str]]- timeout: int | float- init_script: Optional[str]- user_data_dir: str- selector_config: Optional[Dict]- additional_args: Optional[Dict]- locale: Optional[str]- real_chrome: bool- cdp_url: Optional[str]- useragent: Optional[str]- extra_flags: Optional[List[str]]- blocked_domains: Optional[Set[str]]- retries: int- retry_delay: int | float-- class PlaywrightFetchParams(TypedDict, total=False):- load_dom: bool- wait: int | float- network_idle: bool- google_search: bool- timeout: int | float- disable_resources: bool- wait_selector: Optional[str]- page_action: Optional[Callable]- selector_config: Optional[Dict]- extra_headers: Optional[Dict[str, str]]- wait_selector_state: SelectorWaitStates- blocked_domains: Optional[Set[str]]- proxy: Optional[str | Dict[str, str]]-- class StealthSession(PlaywrightSession, total=False):- allow_webgl: bool- hide_canvas: bool- block_webrtc: bool- solve_cloudflare: bool-- class StealthFetchParams(PlaywrightFetchParams, total=False):- solve_cloudflare: bool--else: # pragma: no cover- RequestsSession = TypedDict- GetRequestParams = TypedDict- DataRequestParams = TypedDict- PlaywrightSession = TypedDict- PlaywrightFetchParams = TypedDict- StealthSession = TypedDict- StealthFetchParams = TypedDict+# Types for session initialization+class RequestsSession(TypedDict, total=False):+ impersonate: ImpersonateType+ http3: Optional[bool]+ stealthy_headers: Optional[bool]+ proxies: Optional[ProxySpec]+ proxy: Optional[str]+ proxy_auth: Optional[Tuple[str, str]]+ proxy_rotator: Optional[ProxyRotator]+ timeout: Optional[int | float]+ headers: Optional[Mapping[str, Optional[str]]]+ retries: Optional[int]+ retry_delay: Optional[int]+ follow_redirects: Optional[bool]+ max_redirects: Optional[int]+ verify: Optional[bool]+ cert: Optional[str | Tuple[str, str]]+ selector_config: Optional[Dict]+++# Types for GET request method parameters+class GetRequestParams(RequestsSession, total=False):+ params: Optional[Dict | List | Tuple]+ cookies: Optional[CookieTypes]+ auth: Optional[Tuple[str, str]]+++# Types for POST/PUT/DELETE request method parameters+class DataRequestParams(GetRequestParams, total=False):+ data: Optional[Dict[str, str] | List[Tuple] | str | BytesIO | bytes]+ json: Optional[Dict | List]+++# Types for browser session+class PlaywrightSession(TypedDict, total=False):+ max_pages: int+ headless: bool+ disable_resources: bool+ network_idle: bool+ load_dom: bool+ wait_selector: Optional[str]+ wait_selector_state: SelectorWaitStates+ cookies: Sequence[SetCookieParam] | None+ google_search: bool+ wait: int | float+ timezone_id: str | None+ page_action: Optional[Callable]+ proxy: Optional[str | Dict[str, str] | Tuple]+ proxy_rotator: Optional[ProxyRotator]+ extra_headers: Optional[Dict[str, str]]+ timeout: int | float+ init_script: Optional[str]+ user_data_dir: str+ selector_config: Optional[Dict]+ additional_args: Optional[Dict]+ locale: Optional[str]+ real_chrome: bool+ cdp_url: Optional[str]+ useragent: Optional[str]+ extra_flags: Optional[List[str]]+ blocked_domains: Optional[Set[str]]+ retries: int+ retry_delay: int | float+++class PlaywrightFetchParams(TypedDict, total=False):+ load_dom: bool+ wait: int | float+ network_idle: bool+ google_search: bool+ timeout: int | float+ disable_resources: bool+ wait_selector: Optional[str]+ page_action: Optional[Callable]+ selector_config: Optional[Dict]+ extra_headers: Optional[Dict[str, str]]+ wait_selector_state: SelectorWaitStates+ blocked_domains: Optional[Set[str]]+ proxy: Optional[str | Dict[str, str]]+++class StealthSession(PlaywrightSession, total=False):+ allow_webgl: bool+ hide_canvas: bool+ block_webrtc: bool+ solve_cloudflare: bool+++class StealthFetchParams(PlaywrightFetchParams, total=False):+ solve_cloudflare: bool