Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor/fix(browser fetchers): Solving a logic bug in page rotation

Karim shoairecaec6504959
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
new file mode 100644
index 0000000..55b3310
--- /dev/null
+++ b/scrapling/engines/_browsers/_base.py
@@ -0,0 +1,403 @@
+from time import time, sleep
+from asyncio import sleep as asyncio_sleep, Lock
+
+from camoufox import DefaultAddons
+from playwright.sync_api import BrowserContext, Playwright
+from playwright.async_api import (
+ BrowserContext as AsyncBrowserContext,
+ Playwright as AsyncPlaywright,
+)
+from camoufox.utils import (
+ launch_options as generate_launch_options,
+ installed_verstr as camoufox_version,
+)
+
+from scrapling.engines.toolbelt import (
+ intercept_route,
+ async_intercept_route,
+ get_os_name,
+)
+from ._page import PageInfo, PagePool
+from ._config_tools import _compiled_stealth_scripts
+from ._validators import validate, PlaywrightConfig, CamoufoxConfig
+from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.core._types import (
+ Dict,
+ Optional,
+)
+
+__ff_version_str__ = camoufox_version().split(".", 1)[0]
+
+
+class SyncSession:
+ def __init__(self, max_pages: int = 1):
+ self.max_pages = max_pages
+ self.page_pool = PagePool(max_pages)
+ self.__max_wait_for_page = 60
+ self.playwright: Optional[Playwright] = None
+ self.context: Optional[BrowserContext] = None
+ self._closed = False
+
+ def _get_page(self) -> PageInfo: # pragma: no cover
+ """Get a new page to use"""
+
+ # Close all finished pages to ensure clean state
+ self.page_pool.close_all_finished_pages()
+
+ # If we're at max capacity after cleanup, wait for busy pages to finish
+ if self.page_pool.pages_count >= self.max_pages:
+ start_time = time()
+ while time() - start_time < self.__max_wait_for_page:
+ # Wait for any pages to finish, then clean them up
+ sleep(0.05)
+ self.page_pool.close_all_finished_pages()
+ if self.page_pool.pages_count < self.max_pages:
+ break
+ else:
+ raise TimeoutError(
+ f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period"
+ )
+
+ page = self.context.new_page()
+ timeout = getattr(self, "timeout", 30000)
+ page.set_default_navigation_timeout(timeout)
+ page.set_default_timeout(timeout)
+ if getattr(self, "extra_headers", False):
+ page.set_extra_http_headers(getattr(self, "extra_headers"))
+
+ if getattr(self, "disable_resources", False):
+ page.route("**/*", intercept_route)
+
+ if getattr(self, "stealth", False):
+ for script in _compiled_stealth_scripts():
+ page.add_init_script(script=script)
+
+ return self.page_pool.add_page(page)
+
+ def get_pool_stats(self) -> Dict[str, int]:
+ """Get statistics about the current page pool"""
+ return {
+ "total_pages": self.page_pool.pages_count,
+ "busy_pages": self.page_pool.busy_count,
+ "max_pages": self.max_pages,
+ }
+
+
+class AsyncSession(SyncSession):
+ def __init__(self, max_pages: int = 1):
+ super().__init__(max_pages)
+ self.playwright: Optional[AsyncPlaywright] = None
+ self.context: Optional[AsyncBrowserContext] = None
+ self._lock = Lock()
+
+ async def _get_page(self) -> PageInfo: # pragma: no cover
+ """Get a new page to use"""
+ async with self._lock:
+ # Close all finished pages to ensure clean state
+ await self.page_pool.aclose_all_finished_pages()
+
+ # If we're at max capacity after cleanup, wait for busy pages to finish
+ if self.page_pool.pages_count >= self.max_pages:
+ start_time = time()
+ while time() - start_time < self.__max_wait_for_page:
+ # Wait for any pages to finish, then clean them up
+ await asyncio_sleep(0.05)
+ await self.page_pool.aclose_all_finished_pages()
+ if self.page_pool.pages_count < self.max_pages:
+ break
+ else:
+ raise TimeoutError(
+ f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period"
+ )
+
+ page = await self.context.new_page()
+ timeout = getattr(self, "timeout", 30000)
+ page.set_default_navigation_timeout(timeout)
+ page.set_default_timeout(timeout)
+ if getattr(self, "extra_headers", False):
+ await page.set_extra_http_headers(getattr(self, "extra_headers"))
+
+ if getattr(self, "disable_resources", False):
+ await page.route("**/*", async_intercept_route)
+
+ if getattr(self, "stealth", False):
+ for script in _compiled_stealth_scripts():
+ await page.add_init_script(script=script)
+
+ return self.page_pool.add_page(page)
+
+
+class DynamicSessionMixin:
+ def __validate__(
+ self,
+ __max_pages,
+ headless,
+ google_search,
+ hide_canvas,
+ disable_webgl,
+ real_chrome,
+ stealth,
+ wait,
+ page_action,
+ proxy,
+ locale,
+ extra_headers,
+ useragent,
+ cdp_url,
+ timeout,
+ disable_resources,
+ wait_selector,
+ init_script,
+ cookies,
+ network_idle,
+ wait_selector_state,
+ selector_config,
+ ):
+ params = {
+ "max_pages": __max_pages,
+ "headless": headless,
+ "google_search": google_search,
+ "hide_canvas": hide_canvas,
+ "disable_webgl": disable_webgl,
+ "real_chrome": real_chrome,
+ "stealth": stealth,
+ "wait": wait,
+ "page_action": page_action,
+ "proxy": proxy,
+ "locale": locale,
+ "extra_headers": extra_headers,
+ "useragent": useragent,
+ "timeout": timeout,
+ "selector_config": selector_config,
+ "disable_resources": disable_resources,
+ "wait_selector": wait_selector,
+ "init_script": init_script,
+ "cookies": cookies,
+ "network_idle": network_idle,
+ "wait_selector_state": wait_selector_state,
+ "cdp_url": cdp_url,
+ }
+ config = validate(params, PlaywrightConfig)
+
+ self.max_pages = config.max_pages
+ self.headless = config.headless
+ self.hide_canvas = config.hide_canvas
+ self.disable_webgl = config.disable_webgl
+ self.real_chrome = config.real_chrome
+ self.stealth = config.stealth
+ self.google_search = config.google_search
+ self.wait = config.wait
+ self.proxy = config.proxy
+ self.locale = config.locale
+ self.extra_headers = config.extra_headers
+ self.useragent = config.useragent
+ self.timeout = config.timeout
+ self.cookies = config.cookies
+ self.disable_resources = config.disable_resources
+ self.cdp_url = config.cdp_url
+ self.network_idle = config.network_idle
+ self.wait_selector = config.wait_selector
+ self.init_script = config.init_script
+ self.wait_selector_state = config.wait_selector_state
+ self.selector_config = config.selector_config
+ self.page_action = config.page_action
+ self._headers_keys = (
+ set(map(str.lower, self.extra_headers.keys()))
+ if self.extra_headers
+ else set()
+ )
+ self.__initiate_browser_options__()
+
+ def __initiate_browser_options__(self):
+ if not self.cdp_url:
+ # `launch_options` is used with persistent context
+ self.launch_options = dict(
+ _launch_kwargs(
+ self.headless,
+ self.proxy,
+ self.locale,
+ tuple(self.extra_headers.items())
+ if self.extra_headers
+ else tuple(),
+ self.useragent,
+ self.real_chrome,
+ self.stealth,
+ self.hide_canvas,
+ self.disable_webgl,
+ )
+ )
+ self.launch_options["extra_http_headers"] = dict(
+ self.launch_options["extra_http_headers"]
+ )
+ self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
+ self.context_options = dict()
+ else:
+ # while `context_options` is left to be used when cdp mode is enabled
+ self.launch_options = dict()
+ self.context_options = dict(
+ _context_kwargs(
+ self.proxy,
+ self.locale,
+ tuple(self.extra_headers.items())
+ if self.extra_headers
+ else tuple(),
+ self.useragent,
+ self.stealth,
+ )
+ )
+ self.context_options["extra_http_headers"] = dict(
+ self.context_options["extra_http_headers"]
+ )
+ self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
+
+
+class StealthySessionMixin:
+ def __validate__(
+ self,
+ max_pages,
+ headless,
+ block_images,
+ disable_resources,
+ block_webrtc,
+ allow_webgl,
+ network_idle,
+ humanize,
+ solve_cloudflare,
+ wait,
+ timeout,
+ page_action,
+ wait_selector,
+ init_script,
+ addons,
+ wait_selector_state,
+ cookies,
+ google_search,
+ extra_headers,
+ proxy,
+ os_randomize,
+ disable_ads,
+ geoip,
+ selector_config,
+ additional_args,
+ ):
+ params = {
+ "max_pages": max_pages,
+ "headless": headless,
+ "block_images": block_images,
+ "disable_resources": disable_resources,
+ "block_webrtc": block_webrtc,
+ "allow_webgl": allow_webgl,
+ "network_idle": network_idle,
+ "humanize": humanize,
+ "solve_cloudflare": solve_cloudflare,
+ "wait": wait,
+ "timeout": timeout,
+ "page_action": page_action,
+ "wait_selector": wait_selector,
+ "init_script": init_script,
+ "addons": addons,
+ "wait_selector_state": wait_selector_state,
+ "cookies": cookies,
+ "google_search": google_search,
+ "extra_headers": extra_headers,
+ "proxy": proxy,
+ "os_randomize": os_randomize,
+ "disable_ads": disable_ads,
+ "geoip": geoip,
+ "selector_config": selector_config,
+ "additional_args": additional_args,
+ }
+ config = validate(params, CamoufoxConfig)
+
+ self.max_pages = config.max_pages
+ self.headless = config.headless
+ self.block_images = config.block_images
+ self.disable_resources = config.disable_resources
+ self.block_webrtc = config.block_webrtc
+ self.allow_webgl = config.allow_webgl
+ self.network_idle = config.network_idle
+ self.humanize = config.humanize
+ self.solve_cloudflare = config.solve_cloudflare
+ self.wait = config.wait
+ self.timeout = config.timeout
+ self.page_action = config.page_action
+ self.wait_selector = config.wait_selector
+ self.init_script = config.init_script
+ self.addons = config.addons
+ self.wait_selector_state = config.wait_selector_state
+ self.cookies = config.cookies
+ self.google_search = config.google_search
+ self.extra_headers = config.extra_headers
+ self.proxy = config.proxy
+ self.os_randomize = config.os_randomize
+ self.disable_ads = config.disable_ads
+ self.geoip = config.geoip
+ self.selector_config = config.selector_config
+ self.additional_args = config.additional_args
+ self.selector_config = config.selector_config
+ self.page_action = config.page_action
+ self._headers_keys = (
+ set(map(str.lower, self.extra_headers.keys()))
+ if self.extra_headers
+ else set()
+ )
+ self.__initiate_browser_options__()
+
+ def __initiate_browser_options__(self):
+ """Initiate browser options."""
+ self.launch_options = generate_launch_options(
+ **{
+ "geoip": self.geoip,
+ "proxy": dict(self.proxy) if self.proxy else self.proxy,
+ "addons": self.addons,
+ "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
+ "headless": self.headless,
+ "humanize": True if self.solve_cloudflare else self.humanize,
+ "i_know_what_im_doing": True, # To turn warnings off with the user configurations
+ "allow_webgl": self.allow_webgl,
+ "block_webrtc": self.block_webrtc,
+ "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
+ "os": None if self.os_randomize else get_os_name(),
+ "user_data_dir": "",
+ "ff_version": __ff_version_str__,
+ "firefox_user_prefs": {
+ # This is what enabling `enable_cache` does internally, so we do it from here instead
+ "browser.sessionhistory.max_entries": 10,
+ "browser.sessionhistory.max_total_viewers": -1,
+ "browser.cache.memory.enable": True,
+ "browser.cache.disk_cache_ssl": True,
+ "browser.cache.disk.smart_size.enabled": True,
+ },
+ **self.additional_args,
+ }
+ )
+
+ @staticmethod
+ def _detect_cloudflare(page_content: str) -> str | None:
+ """
+ Detect the type of Cloudflare challenge present in the provided page content.
+
+ This function analyzes the given page content to identify whether a specific
+ type of Cloudflare challenge is present. It checks for three predefined
+ challenge types: non-interactive, managed, and interactive. If a challenge
+ type is detected, it returns the corresponding type as a string. If no
+ challenge type is detected, it returns None.
+
+ Args:
+ page_content (str): The content of the page to analyze for Cloudflare
+ challenge types.
+
+ Returns:
+ str: A string representing the detected Cloudflare challenge type, if
+ found. Returns None if no challenge matches.
+ """
+ challenge_types = (
+ "non-interactive",
+ "managed",
+ "interactive",
+ )
+ for ctype in challenge_types:
+ if f"cType: '{ctype}'" in page_content:
+ return ctype
+
+ return None
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 5afccb1..cd814c1 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -1,17 +1,8 @@
-from time import time, sleep
from re import compile as re_compile
-from asyncio import sleep as asyncio_sleep, Lock
-from camoufox import DefaultAddons
-from camoufox.utils import (
- launch_options as generate_launch_options,
- installed_verstr as camoufox_version,
-)
from playwright.sync_api import (
Response as SyncPlaywrightResponse,
sync_playwright,
- BrowserContext,
- Playwright,
Locator,
Page,
)
@@ -24,9 +15,8 @@ from playwright.async_api import (
Page as async_Page,
)
+from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
-from ._page import PageInfo, PagePool
-from ._validators import validate, CamoufoxConfig
from scrapling.core._types import (
Dict,
List,
@@ -37,17 +27,13 @@ from scrapling.core._types import (
from scrapling.engines.toolbelt import (
Response,
ResponseFactory,
- async_intercept_route,
generate_convincing_referer,
- get_os_name,
- intercept_route,
)
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
-__ff_version_str__ = camoufox_version().split(".", 1)[0]
-class StealthySession:
+class StealthySession(StealthySessionMixin, SyncSession):
"""A Stealthy session manager with page pooling."""
__slots__ = (
@@ -87,7 +73,7 @@ class StealthySession:
def __init__(
self,
- max_pages: int = 1,
+ __max_pages: int = 1,
headless: bool = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
@@ -141,107 +127,38 @@ class StealthySession:
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
- :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
- params = {
- "max_pages": max_pages,
- "headless": headless,
- "block_images": block_images,
- "disable_resources": disable_resources,
- "block_webrtc": block_webrtc,
- "allow_webgl": allow_webgl,
- "network_idle": network_idle,
- "humanize": humanize,
- "solve_cloudflare": solve_cloudflare,
- "wait": wait,
- "timeout": timeout,
- "page_action": page_action,
- "wait_selector": wait_selector,
- "init_script": init_script,
- "addons": addons,
- "wait_selector_state": wait_selector_state,
- "cookies": cookies,
- "google_search": google_search,
- "extra_headers": extra_headers,
- "proxy": proxy,
- "os_randomize": os_randomize,
- "disable_ads": disable_ads,
- "geoip": geoip,
- "selector_config": selector_config,
- "additional_args": additional_args,
- }
- config = validate(params, CamoufoxConfig)
-
- self.max_pages = config.max_pages
- self.headless = config.headless
- self.block_images = config.block_images
- self.disable_resources = config.disable_resources
- self.block_webrtc = config.block_webrtc
- self.allow_webgl = config.allow_webgl
- self.network_idle = config.network_idle
- self.humanize = config.humanize
- self.solve_cloudflare = config.solve_cloudflare
- self.wait = config.wait
- self.timeout = config.timeout
- self.page_action = config.page_action
- self.wait_selector = config.wait_selector
- self.init_script = config.init_script
- self.addons = config.addons
- self.wait_selector_state = config.wait_selector_state
- self.cookies = config.cookies
- self.google_search = config.google_search
- self.extra_headers = config.extra_headers
- self.proxy = config.proxy
- self.os_randomize = config.os_randomize
- self.disable_ads = config.disable_ads
- self.geoip = config.geoip
- self.selector_config = config.selector_config
- self.additional_args = config.additional_args
-
- self.playwright: Optional[Playwright] = None
- self.context: Optional[BrowserContext] = None
- self.page_pool = PagePool(self.max_pages)
- self._closed = False
- self.selector_config = config.selector_config
- self.page_action = config.page_action
- self._headers_keys = (
- set(map(str.lower, self.extra_headers.keys()))
- if self.extra_headers
- else set()
- )
- self.__initiate_browser_options__()
-
- def __initiate_browser_options__(self):
- """Initiate browser options."""
- self.launch_options = generate_launch_options(
- **{
- "geoip": self.geoip,
- "proxy": dict(self.proxy) if self.proxy else self.proxy,
- "addons": self.addons,
- "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
- "headless": self.headless,
- "humanize": True if self.solve_cloudflare else self.humanize,
- "i_know_what_im_doing": True, # To turn warnings off with the user configurations
- "allow_webgl": self.allow_webgl,
- "block_webrtc": self.block_webrtc,
- "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
- "os": None if self.os_randomize else get_os_name(),
- "user_data_dir": "",
- "ff_version": __ff_version_str__,
- "firefox_user_prefs": {
- # This is what enabling `enable_cache` does internally, so we do it from here instead
- "browser.sessionhistory.max_entries": 10,
- "browser.sessionhistory.max_total_viewers": -1,
- "browser.cache.memory.enable": True,
- "browser.cache.disk_cache_ssl": True,
- "browser.cache.disk.smart_size.enabled": True,
- },
- **self.additional_args,
- }
+ self.__validate__(
+ __max_pages,
+ headless,
+ block_images,
+ disable_resources,
+ block_webrtc,
+ allow_webgl,
+ network_idle,
+ humanize,
+ solve_cloudflare,
+ wait,
+ timeout,
+ page_action,
+ wait_selector,
+ init_script,
+ addons,
+ wait_selector_state,
+ cookies,
+ google_search,
+ extra_headers,
+ proxy,
+ os_randomize,
+ disable_ads,
+ geoip,
+ selector_config,
+ additional_args,
)
+ super().__init__(max_pages=self.max_pages)
def __create__(self):
"""Create a browser for this instance and context."""
@@ -284,68 +201,6 @@ class StealthySession:
self._closed = True
- def _get_or_create_page(self) -> PageInfo: # pragma: no cover
- """Get an available page or create a new one"""
- # Try to get a ready page first
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
-
- # Create a new page if under limit
- if self.page_pool.pages_count < self.max_pages:
- page = self.context.new_page()
- page.set_default_navigation_timeout(self.timeout)
- page.set_default_timeout(self.timeout)
- if self.extra_headers:
- page.set_extra_http_headers(self.extra_headers)
-
- if self.disable_resources:
- page.route("**/*", intercept_route)
-
- return self.page_pool.add_page(page)
-
- # Wait for a page to become available
- max_wait = 30
- start_time = time()
-
- while time() - start_time < max_wait:
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
- sleep(0.05)
-
- raise TimeoutError("No pages available within timeout period")
-
- @staticmethod
- def _detect_cloudflare(page_content):
- """
- Detect the type of Cloudflare challenge present in the provided page content.
-
- This function analyzes the given page content to identify whether a specific
- type of Cloudflare challenge is present. It checks for three predefined
- challenge types: non-interactive, managed, and interactive. If a challenge
- type is detected, it returns the corresponding type as a string. If no
- challenge type is detected, it returns None.
-
- Args:
- page_content (str): The content of the page to analyze for Cloudflare
- challenge types.
-
- Returns:
- str: A string representing the detected Cloudflare challenge type, if
- found. Returns None if no challenge matches.
- """
- challenge_types = (
- "non-interactive",
- "managed",
- "interactive",
- )
- for ctype in challenge_types:
- if f"cType: '{ctype}'" in page_content:
- return ctype
-
- return None
-
def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
@@ -416,7 +271,7 @@ class StealthySession:
):
final_response = finished_response
- page_info = self._get_or_create_page()
+ page_info = self._get_page()
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -462,8 +317,8 @@ class StealthySession:
page_info.page, first_response, final_response, self.selector_config
)
- # Mark the page as ready for next use
- page_info.mark_ready()
+ # Mark the page as finished for next use
+ page_info.mark_finished()
return response
@@ -471,17 +326,8 @@ class StealthySession:
page_info.mark_error()
raise e
- def get_pool_stats(self) -> Dict[str, int]:
- """Get statistics about the current page pool"""
- return {
- "total_pages": self.page_pool.pages_count,
- "ready_pages": self.page_pool.ready_count,
- "busy_pages": self.page_pool.busy_count,
- "max_pages": self.max_pages,
- }
-
-class AsyncStealthySession(StealthySession):
+class AsyncStealthySession(StealthySessionMixin, AsyncSession):
"""A Stealthy session manager with page pooling."""
def __init__(
@@ -544,7 +390,7 @@ class AsyncStealthySession(StealthySession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.
"""
- super().__init__(
+ self.__validate__(
max_pages,
headless,
block_images,
@@ -571,11 +417,7 @@ class AsyncStealthySession(StealthySession):
selector_config,
additional_args,
)
- self.playwright: Optional[AsyncPlaywright] = None
- self.context: Optional[AsyncBrowserContext] = None
- self._lock = Lock()
- self.__enter__ = None
- self.__exit__ = None
+ super().__init__(max_pages=self.max_pages)
async def __create__(self):
"""Create a browser for this instance and context."""
@@ -618,39 +460,6 @@ class AsyncStealthySession(StealthySession):
self._closed = True
- async def _get_or_create_page(self) -> PageInfo:
- """Get an available page or create a new one"""
- async with self._lock:
- # Try to get a ready page first
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
-
- # Create a new page if under limit
- if self.page_pool.pages_count < self.max_pages:
- page = await self.context.new_page()
- page.set_default_navigation_timeout(self.timeout)
- page.set_default_timeout(self.timeout)
- if self.extra_headers:
- await page.set_extra_http_headers(self.extra_headers)
-
- if self.disable_resources:
- await page.route("**/*", async_intercept_route)
-
- return self.page_pool.add_page(page)
-
- # Wait for a page to become available
- max_wait = 30
- start_time = time()
-
- while time() - start_time < max_wait: # pragma: no cover
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
- await asyncio_sleep(0.05)
-
- raise TimeoutError("No pages available within timeout period")
-
async def _solve_cloudflare(self, page: async_Page):
"""Solve the cloudflare challenge displayed on the playwright page passed. The async version
@@ -723,7 +532,7 @@ class AsyncStealthySession(StealthySession):
):
final_response = finished_response
- page_info = await self._get_or_create_page()
+ page_info = await self._get_page()
page_info.mark_busy(url=url)
try:
@@ -771,8 +580,8 @@ class AsyncStealthySession(StealthySession):
page_info.page, first_response, final_response, self.selector_config
)
- # Mark the page as ready for next use
- page_info.mark_ready()
+ # Mark the page as finished for next use
+ page_info.mark_finished()
return response
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 85db198..ef3b906 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -1,10 +1,6 @@
-from time import time, sleep
-from asyncio import sleep as asyncio_sleep, Lock
-
from playwright.sync_api import (
Response as SyncPlaywrightResponse,
sync_playwright,
- BrowserContext,
Playwright,
Locator,
)
@@ -21,9 +17,7 @@ from rebrowser_playwright.async_api import (
)
from scrapling.core.utils import log
-from ._page import PageInfo, PagePool
-from ._validators import validate, PlaywrightConfig
-from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
+from ._base import SyncSession, AsyncSession, DynamicSessionMixin
from scrapling.core._types import (
Dict,
List,
@@ -35,12 +29,10 @@ from scrapling.engines.toolbelt import (
Response,
ResponseFactory,
generate_convincing_referer,
- intercept_route,
- async_intercept_route,
)
-class DynamicSession:
+class DynamicSession(DynamicSessionMixin, SyncSession):
"""A Browser session manager with page pooling."""
__slots__ = (
@@ -127,108 +119,31 @@ class DynamicSession:
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
-
- params = {
- "max_pages": __max_pages,
- "headless": headless,
- "google_search": google_search,
- "hide_canvas": hide_canvas,
- "disable_webgl": disable_webgl,
- "real_chrome": real_chrome,
- "stealth": stealth,
- "wait": wait,
- "page_action": page_action,
- "proxy": proxy,
- "locale": locale,
- "extra_headers": extra_headers,
- "useragent": useragent,
- "timeout": timeout,
- "selector_config": selector_config,
- "disable_resources": disable_resources,
- "wait_selector": wait_selector,
- "init_script": init_script,
- "cookies": cookies,
- "network_idle": network_idle,
- "wait_selector_state": wait_selector_state,
- "cdp_url": cdp_url,
- }
- config = validate(params, PlaywrightConfig)
-
- self.max_pages = config.max_pages
- self.headless = config.headless
- self.hide_canvas = config.hide_canvas
- self.disable_webgl = config.disable_webgl
- self.real_chrome = config.real_chrome
- self.stealth = config.stealth
- self.google_search = config.google_search
- self.wait = config.wait
- self.proxy = config.proxy
- self.locale = config.locale
- self.extra_headers = config.extra_headers
- self.useragent = config.useragent
- self.timeout = config.timeout
- self.cookies = config.cookies
- self.disable_resources = config.disable_resources
- self.cdp_url = config.cdp_url
- self.network_idle = config.network_idle
- self.wait_selector = config.wait_selector
- self.init_script = config.init_script
- self.wait_selector_state = config.wait_selector_state
-
- self.playwright: Optional[Playwright] = None
- self.context: Optional[BrowserContext] = None
- self.page_pool = PagePool(self.max_pages)
- self._closed = False
- self.selector_config = config.selector_config
- self.page_action = config.page_action
- self._headers_keys = (
- set(map(str.lower, self.extra_headers.keys()))
- if self.extra_headers
- else set()
+ self.__validate__(
+ __max_pages,
+ headless,
+ google_search,
+ hide_canvas,
+ disable_webgl,
+ real_chrome,
+ stealth,
+ wait,
+ page_action,
+ proxy,
+ locale,
+ extra_headers,
+ useragent,
+ cdp_url,
+ timeout,
+ disable_resources,
+ wait_selector,
+ init_script,
+ cookies,
+ network_idle,
+ wait_selector_state,
+ selector_config,
)
- self.__initiate_browser_options__()
-
- def __initiate_browser_options__(self):
- if not self.cdp_url:
- # `launch_options` is used with persistent context
- self.launch_options = dict(
- _launch_kwargs(
- self.headless,
- self.proxy,
- self.locale,
- tuple(self.extra_headers.items())
- if self.extra_headers
- else tuple(),
- self.useragent,
- self.real_chrome,
- self.stealth,
- self.hide_canvas,
- self.disable_webgl,
- )
- )
- self.launch_options["extra_http_headers"] = dict(
- self.launch_options["extra_http_headers"]
- )
- self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
- self.context_options = dict()
- else:
- # while `context_options` is left to be used when cdp mode is enabled
- self.launch_options = dict()
- self.context_options = dict(
- _context_kwargs(
- self.proxy,
- self.locale,
- tuple(self.extra_headers.items())
- if self.extra_headers
- else tuple(),
- self.useragent,
- self.stealth,
- )
- )
- self.context_options["extra_http_headers"] = dict(
- self.context_options["extra_http_headers"]
- )
- self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
+ super().__init__(max_pages=self.max_pages)
def __create__(self):
"""Create a browser for this instance and context."""
@@ -237,7 +152,7 @@ class DynamicSession:
# Because rebrowser_playwright doesn't play well with real browsers
sync_context = sync_playwright
- self.playwright = sync_context().start()
+ self.playwright: Playwright = sync_context().start()
if self.cdp_url: # pragma: no cover
self.context = self.playwright.chromium.connect_over_cdp(
@@ -280,43 +195,10 @@ class DynamicSession:
self._closed = True
- def _get_or_create_page(self) -> PageInfo: # pragma: no cover
- """Get an available page or create a new one"""
- # Try to get a ready page first
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
-
- # Create a new page if under limit
- if self.page_pool.pages_count < self.max_pages:
- page = self.context.new_page()
- page.set_default_navigation_timeout(self.timeout)
- page.set_default_timeout(self.timeout)
- if self.extra_headers:
- page.set_extra_http_headers(self.extra_headers)
-
- if self.disable_resources:
- page.route("**/*", intercept_route)
-
- if self.stealth:
- for script in _compiled_stealth_scripts():
- page.add_init_script(script=script)
-
- return self.page_pool.add_page(page)
-
- # Wait for a page to become available
- max_wait = 30
- start_time = time()
-
- while time() - start_time < max_wait:
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
- sleep(0.05)
-
- raise TimeoutError("No pages available within timeout period")
-
- def fetch(self, url: str) -> Response:
+ def fetch(
+ self,
+ url: str,
+ ) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
@@ -340,7 +222,7 @@ class DynamicSession:
):
final_response = finished_response
- page_info = self._get_or_create_page()
+ page_info = self._get_page()
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -380,8 +262,8 @@ class DynamicSession:
page_info.page, first_response, final_response, self.selector_config
)
- # Mark the page as ready for next use
- page_info.mark_ready()
+ # Mark the page as finished for next use
+ page_info.mark_finished()
return response
@@ -389,17 +271,8 @@ class DynamicSession:
page_info.mark_error()
raise e
- def get_pool_stats(self) -> Dict[str, int]:
- """Get statistics about the current page pool"""
- return {
- "total_pages": self.page_pool.pages_count,
- "ready_pages": self.page_pool.ready_count,
- "busy_pages": self.page_pool.busy_count,
- "max_pages": self.max_pages,
- }
-
-class AsyncDynamicSession(DynamicSession):
+class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
def __init__(
@@ -455,7 +328,7 @@ class AsyncDynamicSession(DynamicSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
"""
- super().__init__(
+ self.__validate__(
max_pages,
headless,
google_search,
@@ -479,12 +352,7 @@ class AsyncDynamicSession(DynamicSession):
wait_selector_state,
selector_config,
)
-
- self.playwright: Optional[AsyncPlaywright] = None
- self.context: Optional[AsyncBrowserContext] = None
- self._lock = Lock()
- self.__enter__ = None
- self.__exit__ = None
+ super().__init__(max_pages=self.max_pages)
async def __create__(self):
"""Create a browser for this instance and context."""
@@ -541,43 +409,6 @@ class AsyncDynamicSession(DynamicSession):
self._closed = True
- async def _get_or_create_page(self) -> PageInfo:
- """Get an available page or create a new one"""
- async with self._lock:
- # Try to get a ready page first
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
-
- # Create a new page if under limit
- if self.page_pool.pages_count < self.max_pages:
- page = await self.context.new_page()
- page.set_default_navigation_timeout(self.timeout)
- page.set_default_timeout(self.timeout)
- if self.extra_headers:
- await page.set_extra_http_headers(self.extra_headers)
-
- if self.disable_resources:
- await page.route("**/*", async_intercept_route)
-
- if self.stealth:
- for script in _compiled_stealth_scripts():
- await page.add_init_script(script=script)
-
- return self.page_pool.add_page(page)
-
- # Wait for a page to become available
- max_wait = 30 # seconds
- start_time = time()
-
- while time() - start_time < max_wait: # pragma: no cover
- page_info = self.page_pool.get_ready_page()
- if page_info:
- return page_info
- await asyncio_sleep(0.05)
-
- raise TimeoutError("No pages available within timeout period")
-
async def fetch(self, url: str) -> Response:
"""Opens up the browser and do your request based on your chosen options.
@@ -602,7 +433,7 @@ class AsyncDynamicSession(DynamicSession):
):
final_response = finished_response
- page_info = await self._get_or_create_page()
+ page_info = await self._get_page()
page_info.mark_busy(url=url)
try:
@@ -642,8 +473,8 @@ class AsyncDynamicSession(DynamicSession):
page_info.page, first_response, final_response, self.selector_config
)
- # Mark the page as ready for next use
- page_info.mark_ready()
+ # Mark the page as finished for next use
+ page_info.mark_finished()
return response
diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py
index ec418d0..61dd51b 100644
--- a/scrapling/engines/_browsers/_page.py
+++ b/scrapling/engines/_browsers/_page.py
@@ -6,7 +6,9 @@ from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal
-PageState = Literal["ready", "busy", "error"] # States that a page can be in
+PageState = Literal[
+ "finished", "ready", "busy", "error"
+] # States that a page can be in
@dataclass
@@ -23,9 +25,9 @@ class PageInfo:
self.state = "busy"
self.url = url
- def mark_ready(self):
- """Mark the page as ready for new requests"""
- self.state = "ready"
+ def mark_finished(self):
+ """Mark the page as finished for new requests"""
+ self.state = "finished"
self.url = ""
def mark_error(self):
@@ -62,24 +64,16 @@ class PagePool:
self.pages.append(page_info)
return page_info
- def get_ready_page(self) -> Optional[PageInfo]:
- """Get a page that's ready for use"""
- with self._lock:
- for page_info in self.pages:
- if page_info.state == "ready":
- return page_info
- return None
-
@property
def pages_count(self) -> int:
"""Get the total number of pages"""
return len(self.pages)
@property
- def ready_count(self) -> int:
- """Get the number of ready pages"""
+ def finished_count(self) -> int:
+ """Get the number of finished pages"""
with self._lock:
- return sum(1 for p in self.pages if p.state == "ready")
+ return sum(1 for p in self.pages if p.state == "finished")
@property
def busy_count(self) -> int:
@@ -91,3 +85,33 @@ class PagePool:
"""Remove pages in error state"""
with self._lock:
self.pages = [p for p in self.pages if p.state != "error"]
+
+ def close_all_finished_pages(self):
+ """Close all pages in finished state and remove them from the pool"""
+ with self._lock:
+ pages_to_remove = []
+ for page_info in self.pages:
+ if page_info.state == "finished":
+ try:
+ page_info.page.close()
+ except Exception:
+ pass
+ pages_to_remove.append(page_info)
+
+ for page_info in pages_to_remove:
+ self.pages.remove(page_info)
+
+ async def aclose_all_finished_pages(self):
+ """Async version: Close all pages in finished state and remove them from the pool"""
+ with self._lock:
+ pages_to_remove = []
+ for page_info in self.pages:
+ if page_info.state == "finished":
+ try:
+ await page_info.page.close()
+ except Exception:
+ pass
+ pages_to_remove.append(page_info)
+
+ for page_info in pages_to_remove:
+ self.pages.remove(page_info)
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index c1ce8f7..3e39ba2 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -118,7 +118,6 @@ class StealthyFetcher(BaseFetcher):
with StealthySession(
wait=wait,
- max_pages=1,
proxy=proxy,
geoip=geoip,
addons=addons,