🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(browsers): solving an issues with leaving playwright loop open when cdp connection fails
Karim shoairbf5aa021d4c8
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex bd5d455..a9dbe85 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -3,13 +3,11 @@ from asyncio import sleep as asyncio_sleepfrom playwright.sync_api import (Locator,- Playwright,sync_playwright,)from playwright.async_api import (async_playwright,Locator as AsyncLocator,- Playwright as AsyncPlaywright,BrowserContext as AsyncBrowserContext,)@@ -71,21 +69,27 @@ class DynamicSession(SyncSession, DynamicSessionMixin):def start(self):"""Create a browser for this instance and context."""if not self.playwright:- self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue]+ self.playwright = sync_playwright().start()- if self._config.cdp_url: # pragma: no cover- browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)- self.context = browser.new_context(**self._context_options)- else:- self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)+ try:+ if self._config.cdp_url: # pragma: no cover+ browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)+ self.context = browser.new_context(**self._context_options)+ else:+ self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)- if self._config.init_script: # pragma: no cover- self.context.add_init_script(path=self._config.init_script)+ if self._config.init_script: # pragma: no cover+ self.context.add_init_script(path=self._config.init_script)- if self._config.cookies: # pragma: no cover- self.context.add_cookies(self._config.cookies)+ if self._config.cookies: # pragma: no cover+ self.context.add_cookies(self._config.cookies)- self._is_alive = True+ self._is_alive = True+ except Exception:+ # Clean up playwright if browser setup fails+ self.playwright.stop()+ self.playwright = None+ raiseelse:raise RuntimeError("Session has been already started")@@ -209,23 +213,28 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):async def start(self):"""Create a browser for this instance and context."""if not self.playwright:- self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue]-- if self._config.cdp_url:- browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)- self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)- else:- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(- **self._launch_options- )-- if self._config.init_script: # pragma: no cover- await self.context.add_init_script(path=self._config.init_script)-- if self._config.cookies:- await self.context.add_cookies(self._config.cookies) # pyright: ignore-- self._is_alive = True+ self.playwright = await async_playwright().start()+ try:+ if self._config.cdp_url:+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)+ self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)+ else:+ self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(+ **self._launch_options+ )++ if self._config.init_script: # pragma: no cover+ await self.context.add_init_script(path=self._config.init_script)++ if self._config.cookies:+ await self.context.add_cookies(self._config.cookies) # pyright: ignore++ self._is_alive = True+ except Exception:+ # Clean up playwright if browser setup fails+ await self.playwright.stop()+ self.playwright = None+ raiseelse:raise RuntimeError("Session has been already started")diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 27c4d79..62c6455 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -6,12 +6,10 @@ from asyncio import sleep as asyncio_sleepfrom playwright.sync_api import (Locator,Page,- Playwright,)from playwright.async_api import (Page as async_Page,Locator as AsyncLocator,- Playwright as AsyncPlaywright,BrowserContext as AsyncBrowserContext,)from patchright.sync_api import sync_playwright@@ -82,24 +80,30 @@ class StealthySession(SyncSession, StealthySessionMixin):def start(self):"""Create a browser for this instance and context."""if not self.playwright:- self.playwright: Playwright = sync_playwright().start() # pyright: ignore [reportAttributeAccessIssue]+ self.playwright = sync_playwright().start()- if self._config.cdp_url: # pragma: no cover- browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)- self.context = browser.new_context(**self._context_options)- else:- self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)+ try:+ if self._config.cdp_url: # pragma: no cover+ browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)+ self.context = browser.new_context(**self._context_options)+ else:+ self.context = self.playwright.chromium.launch_persistent_context(**self._launch_options)- for script in _compiled_stealth_scripts():- self.context.add_init_script(script=script)+ for script in _compiled_stealth_scripts():+ self.context.add_init_script(script=script)- if self._config.init_script: # pragma: no cover- self.context.add_init_script(path=self._config.init_script)+ if self._config.init_script: # pragma: no cover+ self.context.add_init_script(path=self._config.init_script)- if self._config.cookies: # pragma: no cover- self.context.add_cookies(self._config.cookies)+ if self._config.cookies: # pragma: no cover+ self.context.add_cookies(self._config.cookies)- self._is_alive = True+ self._is_alive = True+ except Exception:+ # Clean up playwright if browser setup fails+ self.playwright.stop()+ self.playwright = None+ raiseelse:raise RuntimeError("Session has been already started")@@ -308,26 +312,31 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):async def start(self):"""Create a browser for this instance and context."""if not self.playwright:- self.playwright: AsyncPlaywright = await async_playwright().start() # pyright: ignore [reportAttributeAccessIssue]-- if self._config.cdp_url:- browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)- self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)- else:- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(- **self._launch_options- )+ self.playwright = await async_playwright().start()+ try:+ if self._config.cdp_url:+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)+ self.context: AsyncBrowserContext = await browser.new_context(**self._context_options)+ else:+ self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(+ **self._launch_options+ )- for script in _compiled_stealth_scripts():- await self.context.add_init_script(script=script)+ for script in _compiled_stealth_scripts():+ await self.context.add_init_script(script=script)- if self._config.init_script: # pragma: no cover- await self.context.add_init_script(path=self._config.init_script)+ if self._config.init_script: # pragma: no cover+ await self.context.add_init_script(path=self._config.init_script)- if self._config.cookies:- await self.context.add_cookies(self._config.cookies) # pyright: ignore+ if self._config.cookies:+ await self.context.add_cookies(self._config.cookies) # pyright: ignore- self._is_alive = True+ self._is_alive = True+ except Exception:+ # Clean up playwright if browser setup fails+ await self.playwright.stop()+ self.playwright = None+ raiseelse:raise RuntimeError("Session has been already started")