🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(fetchers): rename internal api
Karim shoairb507e4d4a0c1
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 6e5734e..7aa5aeb 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -40,7 +40,7 @@ class SyncSession:self.context: BrowserContext | Any = Noneself._closed = False- def __create__(self):+ def start(self):passdef close(self): # pragma: no cover@@ -59,7 +59,7 @@ class SyncSession:self._closed = Truedef __enter__(self):- self.__create__()+ self.start()return selfdef __exit__(self, exc_type, exc_val, exc_tb):@@ -145,7 +145,7 @@ class AsyncSession:self._closed = Falseself._lock = Lock()- async def __create__(self):+ async def start(self):passasync def close(self):@@ -164,7 +164,7 @@ class AsyncSession:self._closed = Trueasync def __aenter__(self):- await self.__create__()+ await self.start()return selfasync def __aexit__(self, exc_type, exc_val, exc_tb):diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 436a006..e3cbc67 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -102,16 +102,19 @@ class StealthySession(StealthySessionMixin, SyncSession):self.__validate__(**kwargs)super().__init__(max_pages=self._max_pages)- def __create__(self):+ def start(self):"""Create a browser for this instance and context."""- self.playwright = sync_playwright().start()- self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)+ if not self.playwright:+ self.playwright = sync_playwright().start()+ self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)- if self._init_script: # pragma: no cover- self.context.add_init_script(path=self._init_script)+ if self._init_script: # pragma: no cover+ self.context.add_init_script(path=self._init_script)- if self._cookies: # pragma: no cover- self.context.add_cookies(self._cookies)+ if self._cookies: # pragma: no cover+ self.context.add_cookies(self._cookies)+ else:+ raise RuntimeError("Session has been already started")def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed@@ -299,18 +302,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):self.__validate__(**kwargs)super().__init__(max_pages=self._max_pages)- async def __create__(self):+ async def start(self):"""Create a browser for this instance and context."""- self.playwright: AsyncPlaywright = await async_playwright().start()- self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(- **self.launch_options- )+ if not self.playwright:+ self.playwright: AsyncPlaywright = await async_playwright().start()+ self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(+ **self.launch_options+ )- if self._init_script: # pragma: no cover- await self.context.add_init_script(path=self._init_script)+ if self._init_script: # pragma: no cover+ await self.context.add_init_script(path=self._init_script)- if self._cookies:- await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]+ if self._cookies:+ await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]+ else:+ raise RuntimeError("Session has been already started")async def _cloudflare_solver(self, page: async_Page): # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed. The async versiondiff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 9ab0e3c..a338c93 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -95,24 +95,27 @@ class DynamicSession(DynamicSessionMixin, SyncSession):self.__validate__(**kwargs)super().__init__(max_pages=self._max_pages)- def __create__(self):+ def start(self):"""Create a browser for this instance and context."""- sync_context = sync_patchright if self._stealth else sync_playwright+ if not self.playwright:+ sync_context = sync_patchright if self._stealth else sync_playwright- self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]+ self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]- if self._cdp_url: # pragma: no cover- self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(- **self.context_options- )- else:- self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)+ if self._cdp_url: # pragma: no cover+ self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(+ **self.context_options+ )+ else:+ self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)- if self._init_script: # pragma: no cover- self.context.add_init_script(path=self._init_script)+ if self._init_script: # pragma: no cover+ self.context.add_init_script(path=self._init_script)- if self._cookies: # pragma: no cover- self.context.add_cookies(self._cookies)+ if self._cookies: # pragma: no cover+ self.context.add_cookies(self._cookies)+ else:+ raise RuntimeError("Session has been already started")def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -227,25 +230,28 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):self.__validate__(**kwargs)super().__init__(max_pages=self._max_pages)- async def __create__(self):+ async def start(self):"""Create a browser for this instance and context."""- async_context = async_patchright if self._stealth else async_playwright+ if not self.playwright:+ async_context = async_patchright if self._stealth else async_playwright- self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]+ self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]- if self._cdp_url:- browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)- self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)- else:- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(- **self.launch_options- )+ if self._cdp_url:+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)+ self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)+ else:+ self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(+ **self.launch_options+ )- if self._init_script: # pragma: no cover- await self.context.add_init_script(path=self._init_script)+ if self._init_script: # pragma: no cover+ await self.context.add_init_script(path=self._init_script)- if self._cookies:- await self.context.add_cookies(self._cookies) # pyright: ignore+ if self._cookies:+ await self.context.add_cookies(self._cookies) # pyright: ignore+ else:+ raise RuntimeError("Session has been already started")async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.