Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(fetchers): rename internal api

Karim shoairb507e4d4a0c1
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 6e5734e..7aa5aeb 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -40,7 +40,7 @@ class SyncSession:
self.context: BrowserContext | Any = None
self._closed = False
- def __create__(self):
+ def start(self):
pass
def close(self): # pragma: no cover
@@ -59,7 +59,7 @@ class SyncSession:
self._closed = True
def __enter__(self):
- self.__create__()
+ self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
@@ -145,7 +145,7 @@ class AsyncSession:
self._closed = False
self._lock = Lock()
- async def __create__(self):
+ async def start(self):
pass
async def close(self):
@@ -164,7 +164,7 @@ class AsyncSession:
self._closed = True
async def __aenter__(self):
- await self.__create__()
+ await self.start()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 436a006..e3cbc67 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -102,16 +102,19 @@ class StealthySession(StealthySessionMixin, SyncSession):
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
- def __create__(self):
+ def start(self):
"""Create a browser for this instance and context."""
- self.playwright = sync_playwright().start()
- self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
+ if not self.playwright:
+ self.playwright = sync_playwright().start()
+ self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
- if self._init_script: # pragma: no cover
- self.context.add_init_script(path=self._init_script)
+ if self._init_script: # pragma: no cover
+ self.context.add_init_script(path=self._init_script)
- if self._cookies: # pragma: no cover
- self.context.add_cookies(self._cookies)
+ if self._cookies: # pragma: no cover
+ self.context.add_cookies(self._cookies)
+ else:
+ raise RuntimeError("Session has been already started")
def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
@@ -299,18 +302,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
- async def __create__(self):
+ async def start(self):
"""Create a browser for this instance and context."""
- self.playwright: AsyncPlaywright = await async_playwright().start()
- self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
- **self.launch_options
- )
+ if not self.playwright:
+ self.playwright: AsyncPlaywright = await async_playwright().start()
+ self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
+ **self.launch_options
+ )
- if self._init_script: # pragma: no cover
- await self.context.add_init_script(path=self._init_script)
+ if self._init_script: # pragma: no cover
+ await self.context.add_init_script(path=self._init_script)
- if self._cookies:
- await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]
+ if self._cookies:
+ await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]
+ else:
+ raise RuntimeError("Session has been already started")
async def _cloudflare_solver(self, page: async_Page): # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 9ab0e3c..a338c93 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -95,24 +95,27 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
- def __create__(self):
+ def start(self):
"""Create a browser for this instance and context."""
- sync_context = sync_patchright if self._stealth else sync_playwright
+ if not self.playwright:
+ sync_context = sync_patchright if self._stealth else sync_playwright
- self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
+ self.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]
- if self._cdp_url: # pragma: no cover
- self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
- **self.context_options
- )
- else:
- self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
+ if self._cdp_url: # pragma: no cover
+ self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
+ **self.context_options
+ )
+ else:
+ self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
- if self._init_script: # pragma: no cover
- self.context.add_init_script(path=self._init_script)
+ if self._init_script: # pragma: no cover
+ self.context.add_init_script(path=self._init_script)
- if self._cookies: # pragma: no cover
- self.context.add_cookies(self._cookies)
+ if self._cookies: # pragma: no cover
+ self.context.add_cookies(self._cookies)
+ else:
+ raise RuntimeError("Session has been already started")
def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
@@ -227,25 +230,28 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self.__validate__(**kwargs)
super().__init__(max_pages=self._max_pages)
- async def __create__(self):
+ async def start(self):
"""Create a browser for this instance and context."""
- async_context = async_patchright if self._stealth else async_playwright
+ if not self.playwright:
+ async_context = async_patchright if self._stealth else async_playwright
- self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
+ self.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]
- if self._cdp_url:
- browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
- self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
- else:
- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
- **self.launch_options
- )
+ if self._cdp_url:
+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
+ self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
+ else:
+ self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
+ **self.launch_options
+ )
- if self._init_script: # pragma: no cover
- await self.context.add_init_script(path=self._init_script)
+ if self._init_script: # pragma: no cover
+ await self.context.add_init_script(path=self._init_script)
- if self._cookies:
- await self.context.add_cookies(self._cookies) # pyright: ignore
+ if self._cookies:
+ await self.context.add_cookies(self._cookies) # pyright: ignore
+ else:
+ raise RuntimeError("Session has been already started")
async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.