🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style(Fetchers): Setting correct type hint for `wait_selector_state` argument
Karim shoair5be8c2a95edf
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex 8816b90..84e9a51 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -5,6 +5,8 @@ Type definitions for type checking purposes.from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,List, Literal, Optional, Pattern, Tuple, Type, Union)+SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]+try:from typing import Protocolexcept ImportError:diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 2c0ff3d..f1d8f95 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -3,7 +3,7 @@ from camoufox.async_api import AsyncCamoufoxfrom camoufox.sync_api import Camoufoxfrom scrapling.core._types import (Callable, Dict, List, Literal, Optional,- Union)+ SelectorWaitStates, Union)from scrapling.core.utils import logfrom scrapling.engines.toolbelt import (Response, StatusText,async_intercept_route,@@ -18,7 +18,7 @@ class CamoufoxEngine:self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,+ wait_selector_state: Optional[SelectorWaitStates] = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True,geoip: Optional[bool] = False,adaptor_arguments: Dict = None,diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex e434032..0a6d30d 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -1,6 +1,7 @@import json-from scrapling.core._types import Callable, Dict, Optional, Union+from scrapling.core._types import (Callable, Dict, Optional,+ SelectorWaitStates, Union)from scrapling.core.utils import log, lru_cachefrom scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,NSTBROWSER_DEFAULT_QUERY)@@ -23,7 +24,7 @@ class PlaywrightEngine:page_action: Callable = None,wait_selector: Optional[str] = None,locale: Optional[str] = 'en-US',- wait_selector_state: Optional[str] = 'attached',+ wait_selector_state: SelectorWaitStates = 'attached',stealth: Optional[bool] = False,real_chrome: Optional[bool] = False,hide_canvas: Optional[bool] = False,diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 86f77ae..bdf87a6 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -1,5 +1,5 @@from scrapling.core._types import (Callable, Dict, List, Literal, Optional,- Union)+ SelectorWaitStates, Union)from scrapling.engines import (CamoufoxEngine, PlaywrightEngine, StaticEngine,check_if_engine_usable)from scrapling.engines.toolbelt import BaseFetcher, Response@@ -176,8 +176,8 @@ class StealthyFetcher(BaseFetcher):self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,- os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,+ wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,+ proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,) -> Response:"""Opens up a browser and do your request based on your chosen options below.@@ -234,8 +234,8 @@ class StealthyFetcher(BaseFetcher):self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,- os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,+ wait_selector_state: SelectorWaitStates = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,+ proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True, geoip: Optional[bool] = False,) -> Response:"""Opens up a browser and do your request based on your chosen options below.@@ -308,7 +308,7 @@ class PlayWrightFetcher(BaseFetcher):def fetch(self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,- page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',+ page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,@@ -368,7 +368,7 @@ class PlayWrightFetcher(BaseFetcher):async def async_fetch(self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,- page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',+ page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,