🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor/fix(Playwright engine): Improvements and fixes
Karim shoair0c31fb7c0e6c
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 6b8003c..ce60a51 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -19,20 +19,20 @@ class PlaywrightEngine:self, headless: Union[bool, str] = True,disable_resources: bool = False,useragent: Optional[str] = None,- network_idle: Optional[bool] = False,+ network_idle: bool = False,timeout: Optional[float] = 30000,page_action: Callable = None,wait_selector: Optional[str] = None,locale: Optional[str] = 'en-US',wait_selector_state: SelectorWaitStates = 'attached',- stealth: Optional[bool] = False,- real_chrome: Optional[bool] = False,- hide_canvas: Optional[bool] = False,- disable_webgl: Optional[bool] = False,+ stealth: bool = False,+ real_chrome: bool = False,+ hide_canvas: bool = False,+ disable_webgl: bool = False,cdp_url: Optional[str] = None,- nstbrowser_mode: Optional[bool] = False,+ nstbrowser_mode: bool = False,nstbrowser_config: Optional[Dict] = None,- google_search: Optional[bool] = True,+ google_search: bool = True,extra_headers: Optional[Dict[str, str]] = None,proxy: Optional[Union[str, Dict[str, str]]] = None,adaptor_arguments: Dict = None@@ -188,6 +188,38 @@ class PlaywrightEngine:))+ def _process_response_history(self, first_response):+ """Process response history to build a list of Response objects"""+ history = []+ current_request = first_response.request.redirected_from++ try:+ while current_request:+ try:+ current_response = current_request.response()+ history.insert(0, Response(+ url=current_request.url,+ # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"+ text='',+ body=b'',+ status=current_response.status if current_response else 301,+ reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),+ encoding=current_response.headers.get('content-type', '') or 'utf-8',+ cookies={},+ headers=current_response.all_headers() if current_response else {},+ request_headers=current_request.all_headers(),+ **self.adaptor_arguments+ ))+ except Exception as e:+ log.error(f"Error processing redirect: {e}")+ break++ current_request = current_request.redirected_from+ except Exception as e:+ log.error(f"Error processing response history: {e}")++ return history+def fetch(self, url: str) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -201,8 +233,8 @@ class PlaywrightEngine:else:from rebrowser_playwright.sync_api import sync_playwright- # Store the final responsefinal_response = None+ referer = generate_convincing_referer(url) if self.google_search else Nonedef handle_response(finished_response: PlaywrightResponse):nonlocal final_response@@ -218,11 +250,9 @@ class PlaywrightEngine:browser = p.chromium.launch(**self.__launch_kwargs())context = browser.new_context(**self.__context_kwargs())- # Finally we are in businesspage = context.new_page()page.set_default_navigation_timeout(self.timeout)page.set_default_timeout(self.timeout)- # Listen for all responsespage.on("response", handle_response)if self.extra_headers:@@ -235,54 +265,51 @@ class PlaywrightEngine:for script in self.__stealth_scripts():page.add_init_script(path=script)- first_response = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)+ first_response = page.goto(url, referer=referer)page.wait_for_load_state(state="domcontentloaded")+if self.network_idle:page.wait_for_load_state('networkidle')if self.page_action is not None:- page = self.page_action(page)+ try:+ page = self.page_action(page)+ except Exception as e:+ log.error(f"Error executing page_action: {e}")if self.wait_selector and type(self.wait_selector) is str:- waiter = page.locator(self.wait_selector)- waiter.first.wait_for(state=self.wait_selector_state)- # Wait again after waiting for the selector, helpful with protections like Cloudflare- page.wait_for_load_state(state="load")- page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:- page.wait_for_load_state('networkidle')+ try:+ waiter = page.locator(self.wait_selector)+ waiter.first.wait_for(state=self.wait_selector_state)+ # Wait again after waiting for the selector, helpful with protections like Cloudflare+ page.wait_for_load_state(state="load")+ page.wait_for_load_state(state="domcontentloaded")+ if self.network_idle:+ page.wait_for_load_state('networkidle')+ except Exception as e:+ log.error(f"Error waiting for selector {self.wait_selector}: {e}")# In case we didn't catch a document type somehowfinal_response = final_response if final_response else first_response+ if not final_response:+ raise ValueError("Failed to get a response from the page")+# This will be parsed inside `Response`encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)- history = []- current_request = first_response.request.redirected_from- while current_request:- current_response = current_request.response()-- history.insert(0, Response(- url=current_request.url,- # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"- text='',- body=b'',- status=current_response.status if current_response else 301,- reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),- encoding=current_response.headers.get('content-type', '') or 'utf-8',- cookies={},- headers=current_response.all_headers() if current_response else {},- request_headers=current_request.all_headers(),- **self.adaptor_arguments- ))- current_request = current_request.redirected_from+ history = self._process_response_history(first_response)+ try:+ page_content = page.content()+ except Exception as e:+ log.error(f"Error getting page content: {e}")+ page_content = ""response = Response(url=page.url,- text=page.content(),- body=page.content().encode('utf-8'),+ text=page_content,+ body=page_content.encode('utf-8'),status=final_response.status,reason=status_text,encoding=encoding,@@ -293,6 +320,7 @@ class PlaywrightEngine:**self.adaptor_arguments)page.close()+ context.close()return responseasync def async_fetch(self, url: str) -> Response:@@ -308,8 +336,8 @@ class PlaywrightEngine:else:from rebrowser_playwright.async_api import async_playwright- # Store the final responsefinal_response = None+ referer = generate_convincing_referer(url) if self.google_search else Noneasync def handle_response(finished_response: PlaywrightResponse):nonlocal final_response@@ -325,11 +353,9 @@ class PlaywrightEngine:browser = await p.chromium.launch(**self.__launch_kwargs())context = await browser.new_context(**self.__context_kwargs())- # Finally we are in businesspage = await context.new_page()page.set_default_navigation_timeout(self.timeout)page.set_default_timeout(self.timeout)- # Listen for all responsespage.on("response", handle_response)if self.extra_headers:@@ -342,54 +368,51 @@ class PlaywrightEngine:for script in self.__stealth_scripts():await page.add_init_script(path=script)- first_response = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)+ first_response = await page.goto(url, referer=referer)await page.wait_for_load_state(state="domcontentloaded")+if self.network_idle:await page.wait_for_load_state('networkidle')if self.page_action is not None:- page = await self.page_action(page)+ try:+ page = await self.page_action(page)+ except Exception as e:+ log.error(f"Error executing async page_action: {e}")if self.wait_selector and type(self.wait_selector) is str:- waiter = page.locator(self.wait_selector)- await waiter.first.wait_for(state=self.wait_selector_state)- # Wait again after waiting for the selector, helpful with protections like Cloudflare- await page.wait_for_load_state(state="load")- await page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:- await page.wait_for_load_state('networkidle')+ try:+ waiter = page.locator(self.wait_selector)+ await waiter.first.wait_for(state=self.wait_selector_state)+ # Wait again after waiting for the selector, helpful with protections like Cloudflare+ await page.wait_for_load_state(state="load")+ await page.wait_for_load_state(state="domcontentloaded")+ if self.network_idle:+ await page.wait_for_load_state('networkidle')+ except Exception as e:+ log.error(f"Error waiting for selector {self.wait_selector}: {e}")# In case we didn't catch a document type somehowfinal_response = final_response if final_response else first_response+ if not final_response:+ raise ValueError("Failed to get a response from the page")+# This will be parsed inside `Response`encoding = final_response.headers.get('content-type', '') or 'utf-8' # default encoding# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)- history = []- current_request = first_response.request.redirected_from- while current_request:- current_response = await current_request.response()-- history.insert(0, Response(- url=current_request.url,- # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"- text='',- body=b'',- status=current_response.status if current_response else 301,- reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),- encoding=current_response.headers.get('content-type', '') or 'utf-8',- cookies={},- headers=await current_response.all_headers() if current_response else {},- request_headers=await current_request.all_headers(),- **self.adaptor_arguments- ))- current_request = current_request.redirected_from+ history = self._process_response_history(first_response)+ try:+ page_content = await page.content()+ except Exception as e:+ log.error(f"Error getting page content in async: {e}")+ page_content = ""response = Response(url=page.url,- text=await page.content(),- body=(await page.content()).encode('utf-8'),+ text=page_content,+ body=page_content.encode('utf-8'),status=final_response.status,reason=status_text,encoding=encoding,@@ -400,4 +423,6 @@ class PlaywrightEngine:**self.adaptor_arguments)await page.close()+ await context.close()+return response