Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(controllers): Improve type validation

Karim shoair460b4443c2a4
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 167803a..5148362 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -32,7 +32,7 @@ from scrapling.core._types import (
Dict,
Optional,
Union,
- Iterable,
+ List,
Callable,
SelectorWaitStates,
)
@@ -98,7 +98,7 @@ class DynamicSession:
timeout: Union[int, float] = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
- cookies: Optional[Iterable[Dict]] = None,
+ cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None,
@@ -168,7 +168,7 @@ class DynamicSession:
self.extra_headers = config.extra_headers
self.useragent = config.useragent
self.timeout = config.timeout
- self.cookies = list(config.cookies) if config.cookies else []
+ self.cookies = config.cookies
self.disable_resources = config.disable_resources
self.cdp_url = config.cdp_url
self.network_idle = config.network_idle
@@ -180,7 +180,7 @@ class DynamicSession:
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
- self.adaptor_arguments = config.adaptor_arguments or {}
+ self.adaptor_arguments = config.adaptor_arguments
self.page_action = config.page_action
self.__initiate_browser_options__()
@@ -392,7 +392,7 @@ class AsyncDynamicSession(DynamicSession):
timeout: Union[int, float] = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
- cookies: Optional[Iterable[Dict]] = None,
+ cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None,
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 48a073e..b818aef 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -6,6 +6,8 @@ from scrapling.core._types import (
Union,
Dict,
Callable,
+ Literal,
+ List,
Iterable,
SelectorWaitStates,
)
@@ -34,7 +36,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
timeout: Union[int, float] = 30000
disable_resources: bool = False
wait_selector: Optional[str] = None
- cookies: Optional[Iterable[Dict]] = None
+ cookies: Optional[List[Dict]] = None
network_idle: bool = False
wait_selector_state: SelectorWaitStates = "attached"
adaptor_arguments: Optional[Dict] = None
@@ -43,13 +45,6 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
"""Custom validation after msgspec validation"""
if self.max_pages < 1 or self.max_pages > 50:
raise ValueError("max_pages must be between 1 and 50")
- if self.wait_selector_state not in (
- "attached",
- "detached",
- "hidden",
- "visible",
- ):
- raise ValueError(f"Invalid wait_selector_state: {self.wait_selector_state}")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action is not None and not callable(self.page_action):
@@ -60,6 +55,10 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if self.cdp_url:
self.__validate_cdp(self.cdp_url)
+ if not self.cookies:
+ self.cookies = []
+ if not self.adaptor_arguments:
+ self.adaptor_arguments = {}
@staticmethod
def __validate_cdp(cdp_url):