🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(PlayWrightFetcher): adding the option to `sleep` after fetch before closing the page
Karim shoair128f7b9d6f35
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 0c71fec..5b55a95 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -21,6 +21,7 @@ class PlaywrightEngine:useragent: Optional[str] = None,network_idle: bool = False,timeout: Optional[float] = 30000,+ wait: Optional[int] = 0,page_action: Callable = None,wait_selector: Optional[str] = None,locale: Optional[str] = 'en-US',@@ -46,6 +47,7 @@ class PlaywrightEngine::param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific css selector to be in a specific state.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.@@ -76,6 +78,7 @@ class PlaywrightEngine:self.cdp_url = cdp_urlself.useragent = useragentself.timeout = check_type_validity(timeout, [int, float], 30000)+ self.wait = check_type_validity(wait, [int, float], 0)if page_action is not None:if callable(page_action):self.page_action = page_action@@ -289,6 +292,7 @@ class PlaywrightEngine:except Exception as e:log.error(f"Error waiting for selector {self.wait_selector}: {e}")+ page.wait_for_timeout(self.wait)# In case we didn't catch a document type somehowfinal_response = final_response if final_response else first_responseif not final_response:@@ -392,6 +396,7 @@ class PlaywrightEngine:except Exception as e:log.error(f"Error waiting for selector {self.wait_selector}: {e}")+ await page.wait_for_timeout(self.wait)# In case we didn't catch a document type somehowfinal_response = final_response if final_response else first_responseif not final_response:diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 3848ca6..84596cd 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -384,7 +384,7 @@ class PlayWrightFetcher(BaseFetcher):@classmethoddef fetch(cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,- useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000,+ useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0,page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',@@ -402,7 +402,8 @@ class PlayWrightFetcher(BaseFetcher):This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific css selector to be in a specific state.@@ -426,6 +427,7 @@ class PlayWrightFetcher(BaseFetcher):ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")engine = PlaywrightEngine(+ wait=wait,proxy=proxy,locale=locale,timeout=timeout,@@ -452,7 +454,7 @@ class PlayWrightFetcher(BaseFetcher):@classmethodasync def async_fetch(cls, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,- useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000,+ useragent: Optional[str] = None, network_idle: bool = False, timeout: Optional[float] = 30000, wait: Optional[int] = 0,page_action: Optional[Callable] = None, wait_selector: Optional[str] = None, wait_selector_state: SelectorWaitStates = 'attached',hide_canvas: bool = False, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: bool = True,proxy: Optional[Union[str, Dict[str, str]]] = None, locale: Optional[str] = 'en-US',@@ -470,7 +472,8 @@ class PlayWrightFetcher(BaseFetcher):This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific css selector to be in a specific state.@@ -494,6 +497,7 @@ class PlayWrightFetcher(BaseFetcher):ValueError(f"The custom parser config must be of type dictionary, got {cls.__class__}")engine = PlaywrightEngine(+ wait=wait,proxy=proxy,locale=locale,timeout=timeout,