🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(controllers): Improve type validation
Karim shoair460b4443c2a4
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 167803a..5148362 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -32,7 +32,7 @@ from scrapling.core._types import (Dict,Optional,Union,- Iterable,+ List,Callable,SelectorWaitStates,)@@ -98,7 +98,7 @@ class DynamicSession:timeout: Union[int, float] = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,- cookies: Optional[Iterable[Dict]] = None,+ cookies: Optional[List[Dict]] = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",adaptor_arguments: Optional[Dict] = None,@@ -168,7 +168,7 @@ class DynamicSession:self.extra_headers = config.extra_headersself.useragent = config.useragentself.timeout = config.timeout- self.cookies = list(config.cookies) if config.cookies else []+ self.cookies = config.cookiesself.disable_resources = config.disable_resourcesself.cdp_url = config.cdp_urlself.network_idle = config.network_idle@@ -180,7 +180,7 @@ class DynamicSession:self.context: Optional[BrowserContext] = Noneself.page_pool = PagePool(self.max_pages)self._closed = False- self.adaptor_arguments = config.adaptor_arguments or {}+ self.adaptor_arguments = config.adaptor_argumentsself.page_action = config.page_actionself.__initiate_browser_options__()@@ -392,7 +392,7 @@ class AsyncDynamicSession(DynamicSession):timeout: Union[int, float] = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,- cookies: Optional[Iterable[Dict]] = None,+ cookies: Optional[List[Dict]] = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",adaptor_arguments: Optional[Dict] = None,diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 48a073e..b818aef 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -6,6 +6,8 @@ from scrapling.core._types import (Union,Dict,Callable,+ Literal,+ List,Iterable,SelectorWaitStates,)@@ -34,7 +36,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):timeout: Union[int, float] = 30000disable_resources: bool = Falsewait_selector: Optional[str] = None- cookies: Optional[Iterable[Dict]] = None+ cookies: Optional[List[Dict]] = Nonenetwork_idle: bool = Falsewait_selector_state: SelectorWaitStates = "attached"adaptor_arguments: Optional[Dict] = None@@ -43,13 +45,6 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):"""Custom validation after msgspec validation"""if self.max_pages < 1 or self.max_pages > 50:raise ValueError("max_pages must be between 1 and 50")- if self.wait_selector_state not in (- "attached",- "detached",- "hidden",- "visible",- ):- raise ValueError(f"Invalid wait_selector_state: {self.wait_selector_state}")if self.timeout < 0:raise ValueError("timeout must be >= 0")if self.page_action is not None and not callable(self.page_action):@@ -60,6 +55,10 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)if self.cdp_url:self.__validate_cdp(self.cdp_url)+ if not self.cookies:+ self.cookies = []+ if not self.adaptor_arguments:+ self.adaptor_arguments = {}@staticmethoddef __validate_cdp(cdp_url):