Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(fetchers): Adding the foundation of the new browser-based fetchers logic

Karim shoair145b710960c8
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index da6574a..ee6b5cf 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -24,6 +24,7 @@ from typing import (
SUPPORTED_HTTP_METHODS = Literal["GET", "POST", "PUT", "DELETE"]
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
+PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
StrOrBytes = Union[str, bytes]
try:
diff --git a/scrapling/engines/_browsers/__init__.py b/scrapling/engines/_browsers/__init__.py
new file mode 100644
index 0000000..6554f3c
--- /dev/null
+++ b/scrapling/engines/_browsers/__init__.py
@@ -0,0 +1 @@
+from ._controllers import DynamicSession, AsyncDynamicSession
diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py
new file mode 100644
index 0000000..b63b7f6
--- /dev/null
+++ b/scrapling/engines/_browsers/_config_tools.py
@@ -0,0 +1,99 @@
+from functools import lru_cache
+
+from scrapling.core._types import Tuple
+from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, HARMFUL_DEFAULT_ARGS
+from scrapling.engines.toolbelt import js_bypass_path, generate_headers
+
+__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
+
+
+@lru_cache(1)
+def _compiled_stealth_scripts():
+ """Pre-read and compile stealth scripts"""
+ # Basic bypasses nothing fancy as I'm still working on it
+ # But with adding these bypasses to the above config, it bypasses many online tests like
+ # https://bot.sannysoft.com/
+ # https://kaliiiiiiiiii.github.io/brotector/
+ # https://pixelscan.net/
+ # https://iphey.com/
+ # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint
+ # https://arh.antoinevastel.com/bots/areyouheadless/
+ # https://prescience-data.github.io/execution-monitor.html
+ stealth_scripts_paths = tuple(
+ js_bypass_path(script)
+ for script in (
+ # Order is important
+ "webdriver_fully.js",
+ "window_chrome.js",
+ "navigator_plugins.js",
+ "pdf_viewer.js",
+ "notification_permission.js",
+ "screen_props.js",
+ "playwright_fingerprint.js",
+ )
+ )
+ scripts = []
+ for script_path in stealth_scripts_paths:
+ with open(script_path, "r") as f:
+ scripts.append(f.read())
+ return tuple(scripts)
+
+
+@lru_cache(2, typed=True)
+def _set_flags(hide_canvas, disable_webgl):
+ """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
+ flags = DEFAULT_STEALTH_FLAGS
+ if hide_canvas:
+ flags += ("--fingerprinting-canvas-image-data-noise",)
+ if disable_webgl:
+ flags += (
+ "--disable-webgl",
+ "--disable-webgl-image-chromium",
+ "--disable-webgl2",
+ )
+
+ return flags
+
+
+@lru_cache(2, typed=True)
+def _launch_kwargs(headless, real_chrome, stealth, hide_canvas, disable_webgl) -> Tuple:
+ """Creates the arguments we will use while launching playwright's browser"""
+ launch_kwargs = {
+ "headless": headless,
+ "ignore_default_args": HARMFUL_DEFAULT_ARGS,
+ "channel": "chrome" if real_chrome else "chromium",
+ }
+ if stealth:
+ launch_kwargs.update(
+ {"args": _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True}
+ )
+
+ return tuple(launch_kwargs.items())
+
+
+@lru_cache(2, typed=True)
+def _context_kwargs(proxy, locale, extra_headers, useragent, stealth) -> Tuple:
+ """Creates the arguments for the browser context"""
+ context_kwargs = {
+ "proxy": proxy or tuple(),
+ "locale": locale,
+ "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
+ "device_scale_factor": 2,
+ "extra_http_headers": extra_headers or tuple(),
+ "user_agent": useragent or __default_useragent__,
+ }
+ if stealth:
+ context_kwargs.update(
+ {
+ "is_mobile": False,
+ "has_touch": False,
+ # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
+ "service_workers": "allow",
+ "ignore_https_errors": True,
+ "screen": {"width": 1920, "height": 1080},
+ "viewport": {"width": 1920, "height": 1080},
+ "permissions": ["geolocation", "notifications"],
+ }
+ )
+
+ return tuple(context_kwargs.items())
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
new file mode 100644
index 0000000..607bd83
--- /dev/null
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -0,0 +1,615 @@
+import time
+import asyncio
+
+# from camoufox import AsyncNewBrowser, NewBrowser
+from playwright.sync_api import (
+ sync_playwright,
+ BrowserType,
+ Browser,
+ BrowserContext,
+ Playwright,
+ Locator,
+)
+from playwright.async_api import (
+ async_playwright,
+ BrowserType as AsyncBrowserType,
+ Browser as AsyncBrowser,
+ BrowserContext as AsyncBrowserContext,
+ Playwright as AsyncPlaywright,
+ Locator as AsyncLocator,
+)
+from playwright.sync_api import Response as SyncPlaywrightResponse
+from playwright.async_api import Response as AsyncPlaywrightResponse
+from rebrowser_playwright.sync_api import sync_playwright as sync_rebrowser_playwright
+from rebrowser_playwright.async_api import (
+ async_playwright as async_rebrowser_playwright,
+)
+
+from scrapling.core.utils import log
+from ._page import PageInfo, PagePool
+from ._validators import validate, PlaywrightConfig
+from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
+from scrapling.core._types import (
+ Dict,
+ Optional,
+ Union,
+ Iterable,
+ Callable,
+ SelectorWaitStates,
+)
+from scrapling.engines.toolbelt import (
+ Response,
+ ResponseFactory,
+ generate_convincing_referer,
+ intercept_route,
+ async_intercept_route,
+)
+
+
+class DynamicSession:
+ """A Browser session manager with page pooling."""
+
+ __slots__ = (
+ "max_pages",
+ "headless",
+ "hide_canvas",
+ "disable_webgl",
+ "real_chrome",
+ "stealth",
+ "google_search",
+ "proxy",
+ "locale",
+ "extra_headers",
+ "useragent",
+ "timeout",
+ "cookies",
+ "disable_resources",
+ "network_idle",
+ "wait_selector",
+ "wait_selector_state",
+ "wait",
+ "playwright",
+ "browser",
+ "context",
+ "page_pool",
+ "_closed",
+ "adaptor_arguments",
+ "page_action",
+ "launch_options",
+ "context_options",
+ "cdp_url",
+ )
+
+ def __init__(
+ self,
+ max_pages: int = 1,
+ headless: bool = True,
+ google_search: bool = True,
+ hide_canvas: bool = False,
+ disable_webgl: bool = False,
+ real_chrome: bool = False,
+ stealth: bool = False,
+ wait: Union[int, float] = 0,
+ page_action: Optional[Callable] = None,
+ proxy: Optional[Union[str, Dict[str, str]]] = None,
+ locale: str = "en-US",
+ extra_headers: Optional[Dict[str, str]] = None,
+ useragent: Optional[str] = None,
+ cdp_url: Optional[str] = None,
+ timeout: Union[int, float] = 30000,
+ disable_resources: bool = False,
+ wait_selector: Optional[str] = None,
+ cookies: Optional[Iterable[Dict]] = None,
+ network_idle: bool = False,
+ wait_selector_state: SelectorWaitStates = "attached",
+ adaptor_arguments: Optional[Dict] = None,
+ ):
+ """A Browser session manager with page pooling
+
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+ :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+ :param cookies: Set cookies for the next request.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+ :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+ :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
+ :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+ :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+ :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+ :param max_pages: The maximum number of pages to be opened at the same time. It will be used in rotation through a PagePool.
+ :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+ """
+
+ params = {
+ "max_pages": max_pages,
+ "headless": headless,
+ "google_search": google_search,
+ "hide_canvas": hide_canvas,
+ "disable_webgl": disable_webgl,
+ "real_chrome": real_chrome,
+ "stealth": stealth,
+ "wait": wait,
+ "page_action": page_action,
+ "proxy": proxy,
+ "locale": locale,
+ "extra_headers": extra_headers,
+ "useragent": useragent,
+ "timeout": timeout,
+ "adaptor_arguments": adaptor_arguments,
+ "disable_resources": disable_resources,
+ "wait_selector": wait_selector,
+ "cookies": cookies,
+ "network_idle": network_idle,
+ "wait_selector_state": wait_selector_state,
+ "cdp_url": cdp_url,
+ }
+ config = validate(params, PlaywrightConfig)
+
+ self.max_pages = config.max_pages
+ self.headless = config.headless
+ self.hide_canvas = config.hide_canvas
+ self.disable_webgl = config.disable_webgl
+ self.real_chrome = config.real_chrome
+ self.stealth = config.stealth
+ self.google_search = config.google_search
+ self.wait = config.wait
+ self.proxy = config.proxy
+ self.locale = config.locale
+ self.extra_headers = config.extra_headers
+ self.useragent = config.useragent
+ self.timeout = config.timeout
+ self.cookies = list(config.cookies) if config.cookies else []
+ self.disable_resources = config.disable_resources
+ self.cdp_url = config.cdp_url
+ self.network_idle = config.network_idle
+ self.wait_selector = config.wait_selector
+ self.wait_selector_state = config.wait_selector_state
+
+ self.playwright: Optional[Playwright] = None
+ self.browser: Optional[Union[BrowserType, Browser]] = None
+ self.context: Optional[BrowserContext] = None
+ self.page_pool = PagePool(self.max_pages)
+ self._closed = False
+ self.adaptor_arguments = config.adaptor_arguments or {}
+ self.page_action = config.page_action
+ self.__initiate_browser_options__()
+
+ def __initiate_browser_options__(self):
+ self.launch_options = dict(
+ _launch_kwargs(
+ self.headless,
+ self.real_chrome,
+ self.stealth,
+ self.hide_canvas,
+ self.disable_webgl,
+ )
+ )
+ self.context_options = dict(
+ _context_kwargs(
+ self.proxy,
+ self.locale,
+ tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
+ self.useragent,
+ self.stealth,
+ )
+ )
+ self.context_options["extra_http_headers"] = dict(
+ self.context_options["extra_http_headers"]
+ )
+ self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
+
+ def __create__(self):
+ """Create a browser for this instance and context."""
+ sync_context = sync_rebrowser_playwright
+ if not self.stealth or self.real_chrome:
+ # Because rebrowser_playwright doesn't play well with real browsers
+ sync_context = sync_playwright
+
+ self.playwright = sync_context().start()
+
+ browser_launcher = getattr(
+ self.playwright, "chrome" if self.real_chrome else "chromium"
+ )
+ if self.cdp_url:
+ self.browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+ else:
+ self.browser = browser_launcher.launch(**self.launch_options)
+
+ self.context = self.browser.new_context(**self.context_options)
+ if self.cookies:
+ self.context.add_cookies(self.cookies)
+
+ def __enter__(self):
+ self.__create__()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def close(self):
+ """Close all resources"""
+ if self._closed:
+ return
+
+ if self.context:
+ self.context.close()
+ self.context = None
+
+ if self.browser:
+ self.browser.close()
+ self.browser = None
+
+ if self.playwright:
+ self.playwright.stop()
+ self.playwright = None
+
+ self._closed = True
+
+ def _get_or_create_page(self) -> PageInfo:
+ """Get an available page or create a new one"""
+ # Try to get a ready page first
+ page_info = self.page_pool.get_ready_page()
+ if page_info:
+ return page_info
+
+ # Create new page if under limit
+ if self.page_pool.pages_count < self.max_pages:
+ page = self.context.new_page()
+ page.set_default_navigation_timeout(self.timeout)
+ page.set_default_timeout(self.timeout)
+ if self.extra_headers:
+ page.set_extra_http_headers(self.extra_headers)
+
+ if self.disable_resources:
+ page.route("**/*", intercept_route)
+
+ if self.stealth:
+ for script in _compiled_stealth_scripts():
+ page.add_init_script(path=script)
+
+ return self.page_pool.add_page(page)
+
+ # Wait for a page to become available
+ max_wait = 30
+ start_time = time.time()
+
+ while time.time() - start_time < max_wait:
+ page_info = self.page_pool.get_ready_page()
+ if page_info:
+ return page_info
+ time.sleep(0.05)
+
+ raise TimeoutError("No pages available within timeout period")
+
+ def fetch(self, url: str) -> Response:
+ """Opens up the browser and do your request based on your chosen options.
+
+ :param url: The Target url.
+ :return: A `Response` object.
+ """
+ if self._closed:
+ raise RuntimeError("Context manager has been closed")
+
+ final_response = None
+ referer = generate_convincing_referer(url) if self.google_search else None
+
+ def handle_response(finished_response: SyncPlaywrightResponse):
+ nonlocal final_response
+ if (
+ finished_response.request.resource_type == "document"
+ and finished_response.request.is_navigation_request()
+ ):
+ final_response = finished_response
+
+ page_info = self._get_or_create_page()
+ page_info.mark_busy(url=url)
+
+ try:
+ # Navigate to URL and wait for a specified state
+ page_info.page.on("response", handle_response)
+ first_response = page_info.page.goto(url, referer=referer)
+ page_info.page.wait_for_load_state(state="domcontentloaded")
+
+ if self.network_idle:
+ page_info.page.wait_for_load_state("networkidle")
+
+ if not first_response:
+ raise RuntimeError(f"Failed to get response for {url}")
+
+ if self.page_action is not None:
+ try:
+ page_info.page = self.page_action(page_info.page)
+ except Exception as e:
+ log.error(f"Error executing page_action: {e}")
+
+ if self.wait_selector:
+ try:
+ waiter: Locator = page_info.page.locator(self.wait_selector)
+ waiter.first.wait_for(state=self.wait_selector_state)
+ # Wait again after waiting for the selector, helpful with protections like Cloudflare
+ page_info.page.wait_for_load_state(state="load")
+ page_info.page.wait_for_load_state(state="domcontentloaded")
+ if self.network_idle:
+ page_info.page.wait_for_load_state("networkidle")
+ except Exception as e:
+ log.error(f"Error waiting for selector {self.wait_selector}: {e}")
+
+ page_info.page.wait_for_timeout(self.wait)
+
+ # Create response object
+ response = ResponseFactory.from_playwright_response(
+ page_info.page, first_response, final_response, self.adaptor_arguments
+ )
+
+ # Mark page as ready for next use
+ page_info.mark_ready()
+
+ return response
+
+ except Exception as e:
+ page_info.mark_error()
+ raise e
+
+ def get_pool_stats(self) -> Dict[str, int]:
+ """Get statistics about the current page pool"""
+ return {
+ "total_pages": self.page_pool.pages_count,
+ "ready_pages": self.page_pool.ready_count,
+ "busy_pages": self.page_pool.busy_count,
+ "max_pages": self.max_pages,
+ }
+
+
+class AsyncDynamicSession(DynamicSession):
+ """A Browser session manager with page pooling"""
+
+ def __init__(
+ self,
+ max_pages: int = 1,
+ headless: bool = True,
+ google_search: bool = True,
+ hide_canvas: bool = False,
+ disable_webgl: bool = False,
+ real_chrome: bool = False,
+ stealth: bool = False,
+ wait: Union[int, float] = 0,
+ page_action: Optional[Callable] = None,
+ proxy: Optional[Union[str, Dict[str, str]]] = None,
+ locale: str = "en-US",
+ extra_headers: Optional[Dict[str, str]] = None,
+ useragent: Optional[str] = None,
+ cdp_url: Optional[str] = None,
+ timeout: Union[int, float] = 30000,
+ disable_resources: bool = False,
+ wait_selector: Optional[str] = None,
+ cookies: Optional[Iterable[Dict]] = None,
+ network_idle: bool = False,
+ wait_selector_state: SelectorWaitStates = "attached",
+ adaptor_arguments: Optional[Dict] = None,
+ ):
+ """A Browser session manager with page pooling
+
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+ :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
+ :param cookies: Set cookies for the next request.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+ :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+ :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
+ :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+ :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+ :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+ :param max_pages: The maximum number of pages to be opened at the same time. It will be used in rotation through a PagePool.
+ :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+ """
+
+ super().__init__(
+ max_pages,
+ headless,
+ google_search,
+ hide_canvas,
+ disable_webgl,
+ real_chrome,
+ stealth,
+ wait,
+ page_action,
+ proxy,
+ locale,
+ extra_headers,
+ useragent,
+ cdp_url,
+ timeout,
+ disable_resources,
+ wait_selector,
+ cookies,
+ network_idle,
+ wait_selector_state,
+ adaptor_arguments,
+ )
+
+ self.playwright: Optional[AsyncPlaywright] = None
+ self.browser: Optional[Union[AsyncBrowserType, AsyncBrowser]] = None
+ self.context: Optional[AsyncBrowserContext] = None
+ self._lock = asyncio.Lock()
+ self.__enter__ = None
+ self.__exit__ = None
+
+ async def __create__(self):
+ """Create a browser for this instance and context."""
+ async_context = async_rebrowser_playwright
+ if not self.stealth or self.real_chrome:
+ # Because rebrowser_playwright doesn't play well with real browsers
+ async_context = async_playwright
+
+ self.playwright: AsyncPlaywright = await async_context().start()
+
+ browser_launcher: AsyncBrowserType = getattr(
+ self.playwright, "chrome" if self.real_chrome else "chromium"
+ )
+ if self.cdp_url:
+ self.browser = await browser_launcher.connect_over_cdp(
+ endpoint_url=self.cdp_url
+ )
+ else:
+ self.browser = await browser_launcher.launch(**self.launch_options)
+
+ self.context: AsyncBrowserContext = await self.browser.new_context(
+ **self.context_options
+ )
+
+ if self.cookies:
+ await self.context.add_cookies(self.cookies)
+
+ async def __aenter__(self):
+ await self.__create__()
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ await self.close()
+
+ async def close(self):
+ """Close all resources"""
+ if self._closed:
+ return
+
+ if self.context:
+ await self.context.close()
+ self.context = None
+
+ if self.browser:
+ await self.browser.close()
+ self.browser = None
+
+ if self.playwright:
+ await self.playwright.stop()
+ self.playwright = None
+
+ self._closed = True
+
+ async def _get_or_create_page(self) -> PageInfo:
+ """Get an available page or create a new one"""
+ async with self._lock:
+ # Try to get a ready page first
+ page_info = self.page_pool.get_ready_page()
+ if page_info:
+ return page_info
+
+ # Create new page if under limit
+ if self.page_pool.pages_count < self.max_pages:
+ page = await self.context.new_page()
+ page.set_default_navigation_timeout(self.timeout)
+ page.set_default_timeout(self.timeout)
+ if self.extra_headers:
+ await page.set_extra_http_headers(self.extra_headers)
+
+ if self.disable_resources:
+ await page.route("**/*", async_intercept_route)
+
+ if self.stealth:
+ for script in _compiled_stealth_scripts():
+ await page.add_init_script(path=script)
+
+ return self.page_pool.add_page(page)
+
+ # Wait for a page to become available
+ max_wait = 30 # seconds
+ start_time = time.time()
+
+ while time.time() - start_time < max_wait:
+ page_info = self.page_pool.get_ready_page()
+ if page_info:
+ return page_info
+ await asyncio.sleep(0.05)
+
+ raise TimeoutError("No pages available within timeout period")
+
+ async def fetch(self, url: str) -> Response:
+ """Opens up the browser and do your request based on your chosen options.
+
+ :param url: The Target url.
+ :return: A `Response` object.
+ """
+ if self._closed:
+ raise RuntimeError("Context manager has been closed")
+
+ final_response = None
+ referer = generate_convincing_referer(url) if self.google_search else None
+
+ async def handle_response(finished_response: AsyncPlaywrightResponse):
+ nonlocal final_response
+ if (
+ finished_response.request.resource_type == "document"
+ and finished_response.request.is_navigation_request()
+ ):
+ final_response = finished_response
+
+ page_info = await self._get_or_create_page()
+ page_info.mark_busy(url=url)
+
+ try:
+ # Navigate to URL and wait for a specified state
+ page_info.page.on("response", handle_response)
+ first_response = await page_info.page.goto(url, referer=referer)
+ await page_info.page.wait_for_load_state(state="domcontentloaded")
+
+ if self.network_idle:
+ await page_info.page.wait_for_load_state("networkidle")
+
+ if not first_response:
+ raise RuntimeError(f"Failed to get response for {url}")
+
+ if self.page_action is not None:
+ try:
+ page_info.page = await self.page_action(page_info.page)
+ except Exception as e:
+ log.error(f"Error executing page_action: {e}")
+
+ if self.wait_selector:
+ try:
+ waiter: AsyncLocator = page_info.page.locator(self.wait_selector)
+ await waiter.first.wait_for(state=self.wait_selector_state)
+ # Wait again after waiting for the selector, helpful with protections like Cloudflare
+ await page_info.page.wait_for_load_state(state="load")
+ await page_info.page.wait_for_load_state(state="domcontentloaded")
+ if self.network_idle:
+ await page_info.page.wait_for_load_state("networkidle")
+ except Exception as e:
+ log.error(f"Error waiting for selector {self.wait_selector}: {e}")
+
+ await page_info.page.wait_for_timeout(self.wait)
+
+ # Create response object
+ response = await ResponseFactory.from_async_playwright_response(
+ page_info.page, first_response, final_response, self.adaptor_arguments
+ )
+
+ # Mark page as ready for next use
+ page_info.mark_ready()
+
+ return response
+
+ except Exception as e:
+ page_info.mark_error()
+ raise e
diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py
new file mode 100644
index 0000000..ae163da
--- /dev/null
+++ b/scrapling/engines/_browsers/_page.py
@@ -0,0 +1,93 @@
+from threading import RLock
+from dataclasses import dataclass
+
+from playwright.sync_api import Page as SyncPage
+from playwright.async_api import Page as AsyncPage
+
+from scrapling.core._types import Optional, Union, List, Literal
+
+PageState = Literal["ready", "busy", "error"] # States that a page can be in
+
+
+@dataclass
+class PageInfo:
+ """Information about the page and its current state"""
+
+ __slots__ = ("page", "state", "url")
+ page: Union[SyncPage, AsyncPage]
+ state: PageState
+ url: Optional[str]
+
+ def mark_busy(self, url: str = ""):
+ """Mark the page as busy"""
+ self.state = "busy"
+ self.url = url
+
+ def mark_ready(self):
+ """Mark the page as ready for new requests"""
+ self.state = "ready"
+ self.url = ""
+
+ def mark_error(self):
+ """Mark the page as having an error"""
+ self.state = "error"
+
+ def __repr__(self):
+ return f'Page(URL="{self.url!r}", state={self.state!r})'
+
+ def __eq__(self, other_page):
+ """Comparing this page to another page object."""
+ if other_page.__class__ is not self.__class__:
+ return NotImplemented
+ return self.page == other_page.page
+
+
+class PagePool:
+ """Manages a pool of browser pages/tabs with state tracking"""
+
+ __slots__ = ("max_pages", "pages", "_lock")
+
+ def __init__(self, max_pages: int = 5):
+ self.max_pages = max_pages
+ self.pages: List[PageInfo] = []
+ self._lock = RLock()
+
+ def add_page(self, page: Union[SyncPage, AsyncPage]) -> PageInfo:
+ """Add a new page to the pool"""
+ with self._lock:
+ if len(self.pages) >= self.max_pages:
+ raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")
+
+ page_info = PageInfo(page, "ready", "")
+ self.pages.append(page_info)
+ return page_info
+
+ def get_ready_page(self) -> Optional[PageInfo]:
+ """Get a page that's ready for use"""
+ with self._lock:
+ for page_info in self.pages:
+ if page_info.state == "ready":
+ return page_info
+ return None
+
+ @property
+ def pages_count(self) -> int:
+ """Get the total number of pages"""
+ return len(self.pages)
+
+ @property
+ def ready_count(self) -> int:
+ """Get the number of ready pages"""
+ with self._lock:
+ return sum(1 for p in self.pages if p.state == "ready")
+
+ @property
+ def busy_count(self) -> int:
+ """Get the number of busy pages"""
+ with self._lock:
+ return sum(1 for p in self.pages if p.state == "busy")
+
+ def cleanup_error_pages(self):
+ """Remove pages in error state"""
+ with self._lock:
+ self.pages = [p for p in self.pages if p.state != "error"]
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
new file mode 100644
index 0000000..9b1b127
--- /dev/null
+++ b/scrapling/engines/_browsers/_validators.py
@@ -0,0 +1,88 @@
+import msgspec
+from urllib.parse import urlparse
+
+from scrapling.core._types import (
+ Optional,
+ Union,
+ Dict,
+ Callable,
+ Iterable,
+ SelectorWaitStates,
+)
+from scrapling.engines.toolbelt import construct_proxy_dict
+
+
+class PlaywrightConfig(msgspec.Struct, kw_only=True, frozen=False):
+ """Configuration struct for validation"""
+
+ max_pages: int = 1
+ cdp_url: Optional[str] = None
+ headless: bool = True
+ google_search: bool = True
+ hide_canvas: bool = False
+ disable_webgl: bool = False
+ real_chrome: bool = False
+ stealth: bool = False
+ wait: Union[int, float] = 0
+ page_action: Optional[Callable] = None
+ proxy: Optional[Union[str, Dict[str, str]]] = (
+ None # The default value for proxy in Playwright's source is `None`
+ )
+ locale: str = "en-US"
+ extra_headers: Optional[Dict[str, str]] = None
+ useragent: Optional[str] = None
+ timeout: Union[int, float] = 30000
+ disable_resources: bool = False
+ wait_selector: Optional[str] = None
+ cookies: Optional[Iterable[Dict]] = None
+ network_idle: bool = False
+ wait_selector_state: SelectorWaitStates = "attached"
+ adaptor_arguments: Optional[Dict] = None
+
+ def __post_init__(self):
+ """Custom validation after msgspec validation"""
+ if self.max_pages < 1 or self.max_pages > 50:
+ raise ValueError("max_pages must be between 1 and 50")
+ if self.wait_selector_state not in (
+ "attached",
+ "detached",
+ "hidden",
+ "visible",
+ ):
+ raise ValueError(f"Invalid wait_selector_state: {self.wait_selector_state}")
+ if self.timeout < 0:
+ raise ValueError("timeout must be >= 0")
+ if self.page_action is not None and not callable(self.page_action):
+ raise TypeError(
+ f"page_action must be callable, got {type(self.page_action).__name__}"
+ )
+ if self.proxy:
+ self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
+ if self.cdp_url:
+ self.__validate_cdp(self.cdp_url)
+
+ @staticmethod
+ def __validate_cdp(cdp_url):
+ try:
+ # Check the scheme
+ if not cdp_url.startswith(("ws://", "wss://")):
+ raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
+
+ # Validate hostname and port
+ if not urlparse(cdp_url).netloc:
+ raise ValueError("Invalid hostname for the CDP URL")
+
+ except AttributeError as e:
+ raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
+
+ except Exception as e:
+ raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
+
+
+def validate(params, model):
+ try:
+ config = msgspec.convert(params, model)
+ except msgspec.ValidationError as e:
+ raise TypeError(f"Invalid argument type: {e}")
+
+ return config
diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py
index 12e2928..2a84a2d 100644
--- a/scrapling/engines/constants.py
+++ b/scrapling/engines/constants.py
@@ -12,6 +12,15 @@ DEFAULT_DISABLED_RESOURCES = {
"stylesheet",
}
+HARMFUL_DEFAULT_ARGS = (
+ # This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884
+ "--enable-automation",
+ "--disable-popup-blocking",
+ # '--disable-component-update',
+ # '--disable-default-apps',
+ # '--disable-extensions',
+)
+
DEFAULT_STEALTH_FLAGS = (
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable