Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

26.8 KB
Raw Blame
from time import time
from pathlib import Path
from asyncio import sleep as asyncio_sleep, Lock
from contextlib import contextmanager, asynccontextmanager, suppress

from patchright.sync_api import Page
from patchright.sync_api import (
    Frame,
    BrowserContext,
    Response as SyncPlaywrightResponse,
)
from patchright.async_api import Page as AsyncPage
from patchright.async_api import (
    Frame as AsyncFrame,
    Response as AsyncPlaywrightResponse,
    BrowserContext as AsyncBrowserContext,
)

from scrapling.parser import Selector
from scrapling.engines._browsers._page import PageInfo, PagePool
from scrapling.engines._browsers._network import NetworkRecorder
from scrapling.engines._browsers._validators import validate, PlaywrightConfig, StealthConfig
from scrapling.engines._browsers._config_tools import __default_chrome_useragent__, __default_useragent__
from scrapling.engines.toolbelt.navigation import (
    construct_proxy_dict,
    create_intercept_handler,
    create_async_intercept_handler,
)
from scrapling.core._types import (
    Any,
    Awaitable,
    Dict,
    List,
    Set,
    Optional,
    Callable,
    TYPE_CHECKING,
    cast,
    overload,
    Tuple,
    ProxyType,
    Generator,
    AsyncGenerator,
)
from scrapling.engines.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS


class SyncSession:
    _config: "PlaywrightConfig | StealthConfig"
    _context_options: Dict[str, Any]
    if TYPE_CHECKING:
        _build_context_with_proxy: Callable[..., Dict[str, Any]]

    def __init__(self, max_pages: int = 1, record_requests: bool = False, max_recorded_requests: int = 1000):
        self.max_pages = max_pages
        self.network = NetworkRecorder(enabled=record_requests, max_requests=max_recorded_requests)
        self.page_pool = PagePool(max_pages)
        self._max_wait_for_page = 60
        self.playwright: Any = None
        self.context: Any = None
        self.browser: Any = None
        self._is_alive = False

    def start(self) -> None:
        pass

    def save_state(self, path: str | Path) -> None:
        """Save cookies, local storage and IndexedDB to a Playwright-compatible JSON file.

        Requires a started session without proxy rotation. Does not save sessionStorage.
        """
        if not self._is_alive or self.context is None:
            raise RuntimeError("Session state requires a started browser session without proxy rotation")
        self.context.storage_state(path=Path(path), indexed_db=True)

    def load_state(self, path: str | Path) -> None:
        """Replace cookies, local storage and IndexedDB from a Playwright-compatible JSON file.

        Requires a started session without proxy rotation. Fetch or reload pages to use the restored login.
        """
        if not self._is_alive or self.context is None:
            raise RuntimeError("Session state requires a started browser session without proxy rotation")
        self.context.set_storage_state(path)

    def close_pages(self) -> None:
        """Close every open tab in the session's pool. The next request opens a fresh tab."""
        for page_info in self.page_pool.clear():
            with suppress(Exception):
                page_info.page.close()

    def close(self):  # pragma: no cover
        """Close all resources"""
        self.network._detach()
        if not self._is_alive:
            return

        self.close_pages()
        if self.context:
            self.context.close()
            self.context = None

        if self.browser:
            self.browser.close()
            self.browser = None

        if self.playwright:
            self.playwright.stop()
            self.playwright = None  # pyright: ignore

        self._is_alive = False

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def _initialize_context(self, config: PlaywrightConfig | StealthConfig, ctx: BrowserContext) -> BrowserContext:
        """Initialize the browser context."""
        if config.init_script:
            ctx.add_init_script(path=config.init_script)

        if config.cookies:  # pragma: no cover
            ctx.add_cookies(config.cookies)

        self.network._attach(ctx)
        return ctx

    def _get_page(
        self,
        timeout: int | float,
        extra_headers: Optional[Dict[str, str]],
        disable_resources: bool,
        blocked_domains: Optional[Set[str]] = None,
        context: Optional[BrowserContext] = None,
    ) -> PageInfo[Page]:  # pragma: no cover
        """Get a ready page from the pool, or open a new one"""
        page_info = self.page_pool.get_ready_page() if context is None else None
        if page_info is None:
            ctx = context if context is not None else self.context
            assert ctx is not None, "Browser context not initialized"
            page_info = self.page_pool.add_page(ctx.new_page())

        page = cast(Page, page_info.page)
        page.set_default_navigation_timeout(timeout)
        page.set_default_timeout(timeout)
        page.set_extra_http_headers(extra_headers or {})
        page.unroute_all(behavior="ignoreErrors")
        if disable_resources or blocked_domains:
            page.route("**/*", create_intercept_handler(disable_resources, blocked_domains))
        return cast(PageInfo[Page], page_info)

    def get_pool_stats(self) -> Dict[str, int]:
        """Get statistics about the current page pool"""
        return {
            "total_pages": self.page_pool.pages_count,
            "busy_pages": self.page_pool.busy_count,
            "max_pages": self.max_pages,
        }

    @staticmethod
    def _snapshot(
        page: Page, depth: Optional[int] = None, boxes: bool = False, css_selector: Optional[str] = None
    ) -> str:
        """Return an AI ARIA snapshot of an existing browser page.

        :param page: The browser page to snapshot.
        :param depth: Limit the snapshot tree depth. Defaults to no limit.
        :param boxes: Include element bounding boxes in viewport CSS pixels.
        :param css_selector: Snapshot one matching element instead of the whole page.
        """
        return (page.locator(css_selector) if css_selector else page).aria_snapshot(mode="ai", depth=depth, boxes=boxes)

    @staticmethod
    def _wait_for_networkidle(page: Page | Frame, timeout: Optional[int] = None):
        """Wait for the page to become idle (no network activity) even if there are never-ending requests."""
        try:
            page.wait_for_load_state("networkidle", timeout=timeout)
        except Exception:
            pass

    def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):
        page.wait_for_load_state(state="load")
        if load_dom:
            page.wait_for_load_state(state="domcontentloaded")
        if network_idle:
            self._wait_for_networkidle(page)

    @staticmethod
    def _create_response_handler(
        page_info: PageInfo[Page],
        response_container: List,
    ) -> Callable[[SyncPlaywrightResponse], None]:
        """Create a response handler that captures the final navigation response.

        :param page_info: The PageInfo object containing the page
        :param response_container: A list to store the final response (mutable container)
        :return: A callback function for page.on("response", ...)
        """

        def handle_response(finished_response: SyncPlaywrightResponse) -> None:
            if (
                finished_response.request.resource_type == "document"
                and finished_response.request.is_navigation_request()
                and finished_response.request.frame == page_info.page.main_frame
            ):
                response_container[0] = finished_response

        return handle_response

    @contextmanager
    def _page_generator(
        self,
        timeout: int | float,
        extra_headers: Optional[Dict[str, str]],
        disable_resources: bool,
        proxy: Optional[ProxyType] = None,
        blocked_domains: Optional[Set[str]] = None,
    ) -> Generator["PageInfo[Page]", None, None]:
        """Acquire a page - either from persistent context or fresh context with proxy."""
        if proxy:
            # Rotation mode: create fresh context with the provided proxy
            if not self.browser:  # pragma: no cover
                raise RuntimeError("Browser not initialized for proxy rotation mode")
            context_options = self._build_context_with_proxy(proxy)
            context: BrowserContext = self.browser.new_context(**context_options)

            page_info = None
            try:
                context = self._initialize_context(self._config, context)
                page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)
                yield page_info
            finally:
                if page_info is not None:
                    self.page_pool.remove_page(page_info)
                self.network._detach(context)
                context.close()
        else:
            # Standard mode: use PagePool with persistent context
            page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains)
            try:
                yield page_info
            finally:
                if page_info.state == "error" or page_info.page.is_closed():
                    with suppress(Exception):
                        page_info.page.close()
                    self.page_pool.remove_page(page_info)
                else:
                    page_info.mark_ready()


class AsyncSession:
    _config: "PlaywrightConfig | StealthConfig"
    _context_options: Dict[str, Any]
    if TYPE_CHECKING:
        _build_context_with_proxy: Callable[..., Dict[str, Any]]

    def __init__(self, max_pages: int = 1, record_requests: bool = False, max_recorded_requests: int = 1000):
        self.max_pages = max_pages
        self.network = NetworkRecorder(enabled=record_requests, max_requests=max_recorded_requests, asynchronous=True)
        self.page_pool = PagePool(max_pages)
        self._max_wait_for_page = 60
        self.playwright: Any = None
        self.context: Any = None
        self.browser: Any = None
        self._is_alive = False
        self._lock = Lock()

    async def start(self) -> None:
        pass

    async def save_state(self, path: str | Path) -> None:
        """Save cookies, local storage and IndexedDB to a Playwright-compatible JSON file.

        Requires a started session without proxy rotation. Does not save sessionStorage.
        """
        if not self._is_alive or self.context is None:
            raise RuntimeError("Session state requires a started browser session without proxy rotation")
        await self.context.storage_state(path=Path(path), indexed_db=True)

    async def load_state(self, path: str | Path) -> None:
        """Replace cookies, local storage and IndexedDB from a Playwright-compatible JSON file.

        Requires a started session without proxy rotation. Fetch or reload pages to use the restored login.
        """
        if not self._is_alive or self.context is None:
            raise RuntimeError("Session state requires a started browser session without proxy rotation")
        await self.context.set_storage_state(path)

    async def close_pages(self) -> None:
        """Close every open tab in the session's pool. The next request opens a fresh tab."""
        for page_info in self.page_pool.clear():
            with suppress(Exception):
                await cast(AsyncPage, page_info.page).close()

    async def close(self):
        """Close all resources"""
        self.network._detach()
        if not self._is_alive:  # pragma: no cover
            return

        await self.close_pages()
        if self.context:
            await self.context.close()
            self.context = None  # pyright: ignore

        if self.browser:
            await self.browser.close()
            self.browser = None

        if self.playwright:
            await self.playwright.stop()
            self.playwright = None  # pyright: ignore

        self._is_alive = False

    async def __aenter__(self):
        await self.start()
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self.close()

    async def _initialize_context(
        self, config: PlaywrightConfig | StealthConfig, ctx: AsyncBrowserContext
    ) -> AsyncBrowserContext:
        """Initialize the browser context."""
        if config.init_script:  # pragma: no cover
            await ctx.add_init_script(path=config.init_script)

        if config.cookies:  # pragma: no cover
            await ctx.add_cookies(config.cookies)

        self.network._attach(ctx)
        return ctx

    async def _get_page(
        self,
        timeout: int | float,
        extra_headers: Optional[Dict[str, str]],
        disable_resources: bool,
        blocked_domains: Optional[Set[str]] = None,
        context: Optional[AsyncBrowserContext] = None,
    ) -> PageInfo[AsyncPage]:  # pragma: no cover
        """Get a ready page from the pool, or open a new one"""
        ctx = context if context is not None else self.context
        if TYPE_CHECKING:
            assert ctx is not None, "Browser context not initialized"

        async with self._lock:
            page_info = self.page_pool.get_ready_page() if context is None else None
            if page_info is None and context is None and self.page_pool.pages_count >= self.max_pages:
                # At max capacity with the persistent context, so wait for a busy page to become ready
                start_time = time()
                while time() - start_time < self._max_wait_for_page:
                    await asyncio_sleep(0.05)
                    page_info = self.page_pool.get_ready_page()
                    if page_info is not None:
                        break
                else:
                    raise TimeoutError(
                        f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period"
                    )

            if page_info is None:
                page_info = self.page_pool.add_page(await ctx.new_page())

            page = cast(AsyncPage, page_info.page)
            page.set_default_navigation_timeout(timeout)
            page.set_default_timeout(timeout)
            await page.set_extra_http_headers(extra_headers or {})
            await page.unroute_all(behavior="ignoreErrors")
            if disable_resources or blocked_domains:
                await page.route("**/*", create_async_intercept_handler(disable_resources, blocked_domains))
            return cast(PageInfo[AsyncPage], page_info)

    def get_pool_stats(self) -> Dict[str, int]:
        """Get statistics about the current page pool"""
        return {
            "total_pages": self.page_pool.pages_count,
            "busy_pages": self.page_pool.busy_count,
            "max_pages": self.max_pages,
        }

    @staticmethod
    async def _snapshot(
        page: AsyncPage, depth: Optional[int] = None, boxes: bool = False, css_selector: Optional[str] = None
    ) -> str:
        """Return an AI ARIA snapshot of an existing browser page.

        :param page: The browser page to snapshot.
        :param depth: Limit the snapshot tree depth. Defaults to no limit.
        :param boxes: Include element bounding boxes in viewport CSS pixels.
        :param css_selector: Snapshot one matching element instead of the whole page.
        """
        return await (page.locator(css_selector) if css_selector else page).aria_snapshot(
            mode="ai", depth=depth, boxes=boxes
        )

    @staticmethod
    async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None):
        """Wait for the page to become idle (no network activity) even if there are never-ending requests."""
        try:
            await page.wait_for_load_state("networkidle", timeout=timeout)
        except Exception:
            pass

    async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):
        await page.wait_for_load_state(state="load")
        if load_dom:
            await page.wait_for_load_state(state="domcontentloaded")
        if network_idle:
            await self._wait_for_networkidle(page)

    @staticmethod
    def _create_response_handler(
        page_info: PageInfo[AsyncPage],
        response_container: List,
    ) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]:
        """Create an async response handler that captures the final navigation response.

        :param page_info: The PageInfo object containing the page
        :param response_container: A list to store the final response (mutable container)
        :return: A callback function for page.on("response", ...)
        """

        async def handle_response(finished_response: AsyncPlaywrightResponse) -> None:
            if (
                finished_response.request.resource_type == "document"
                and finished_response.request.is_navigation_request()
                and finished_response.request.frame == page_info.page.main_frame
            ):
                response_container[0] = finished_response

        return handle_response

    @asynccontextmanager
    async def _page_generator(
        self,
        timeout: int | float,
        extra_headers: Optional[Dict[str, str]],
        disable_resources: bool,
        proxy: Optional[ProxyType] = None,
        blocked_domains: Optional[Set[str]] = None,
    ) -> AsyncGenerator["PageInfo[AsyncPage]", None]:
        """Acquire a page - either from persistent context or fresh context with proxy."""
        if proxy:
            # Rotation mode: create fresh context with the provided proxy
            if not self.browser:  # pragma: no cover
                raise RuntimeError("Browser not initialized for proxy rotation mode")
            context_options = self._build_context_with_proxy(proxy)
            context: AsyncBrowserContext = await self.browser.new_context(**context_options)

            page_info = None
            try:
                context = await self._initialize_context(self._config, context)
                page_info = await self._get_page(
                    timeout, extra_headers, disable_resources, blocked_domains, context=context
                )
                yield page_info
            finally:
                if page_info is not None:
                    self.page_pool.remove_page(page_info)
                self.network._detach(context)
                await context.close()
        else:
            # Standard mode: use PagePool with persistent context
            page_info = await self._get_page(timeout, extra_headers, disable_resources, blocked_domains)
            try:
                yield page_info
            finally:
                if page_info.state == "error" or page_info.page.is_closed():
                    with suppress(Exception):
                        await page_info.page.close()
                    self.page_pool.remove_page(page_info)
                else:
                    page_info.mark_ready()


class BaseSessionMixin:
    _config: "PlaywrightConfig | StealthConfig"

    @overload
    def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ...

    @overload
    def __validate_routine__(self, params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...

    def __validate_routine__(
        self, params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]
    ) -> PlaywrightConfig | StealthConfig:
        # Dark color scheme bypasses the 'prefersLightColor' check in creepjs
        self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}
        self._browser_options: Dict[str, Any] = {
            "args": DEFAULT_ARGS,
            "ignore_default_args": HARMFUL_ARGS,
        }
        if "__max_pages" in params:
            params["max_pages"] = params.pop("__max_pages")

        config = validate(params, model=model)
        self._headers_keys = (
            {header.lower() for header in config.extra_headers.keys()} if config.extra_headers else set()
        )

        return config

    def __generate_options__(self, extra_flags: Tuple | None = None) -> None:
        config: PlaywrightConfig | StealthConfig = self._config
        self._context_options.update(
            {
                "proxy": config.proxy,
                "timezone_id": config.timezone_id,
                "extra_http_headers": config.extra_headers,
            }
        )
        if config.locale and config.cdp_url:
            # Launch flags can't be set on remote browsers, so the detectable context option is the best effort left
            self._context_options["locale"] = config.locale
        # The default useragent in the headful is always correct now in the current versions of Playwright
        if config.useragent:
            self._context_options["user_agent"] = config.useragent
        elif not config.useragent and config.headless:
            self._context_options["user_agent"] = (
                __default_chrome_useragent__ if config.real_chrome else __default_useragent__
            )

        if not config.cdp_url:
            flags = self._browser_options["args"]
            if config.extra_flags or extra_flags:
                flags = list(set(tuple(flags) + tuple(extra_flags or ()) + tuple(config.extra_flags or ())))

            if config.dns_over_https:
                doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
                if isinstance(flags, list):
                    flags.append(doh_flag)
                else:
                    flags = list(flags) + [doh_flag]

            if config.locale:
                # The context `locale` option patches the main thread only, so Web Workers keep the browser's real
                # language and WAFs like Cloudflare flag the mismatch. Launch flags set it browser-wide instead,
                # so workers, `Intl`, and the `Accept-Language` header all follow natively.
                base_lang = config.locale.split("-")[0].lower()
                accept_lang = f"{config.locale},{base_lang}" if base_lang != config.locale.lower() else config.locale
                flags = (flags if isinstance(flags, list) else list(flags)) + [
                    f"--lang={config.locale}",
                    f"--accept-lang={accept_lang}",
                ]

            self._browser_options.update(
                {
                    "args": flags,
                    "headless": config.headless,
                    "channel": "chrome" if config.real_chrome else "chromium",
                }
            )
            if config.executable_path:
                self._browser_options["executable_path"] = config.executable_path

            self._user_data_dir = config.user_data_dir
        else:
            self._browser_options = {}

        if config.additional_args:
            self._context_options.update(config.additional_args)

    def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]:
        """
        Build context options with a specific proxy for rotation mode.

        :param proxy: Proxy URL string or Playwright-style proxy dict to use for this context.
        :return: Dictionary of context options for browser.new_context().
        """

        context_options = self._context_options.copy()

        # Override proxy if provided
        if proxy:
            context_options["proxy"] = construct_proxy_dict(proxy)

        return context_options


class StealthySessionMixin(BaseSessionMixin):
    def __validate__(self, **params):
        self._config = self.__validate_routine__(params, model=StealthConfig)
        self._context_options.update(
            {
                "is_mobile": False,
                "has_touch": False,
                # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
                "service_workers": "allow",
                "ignore_https_errors": True,
                "screen": {"width": 1920, "height": 1080},
                "viewport": {"width": 1920, "height": 1080},
                "permissions": ["geolocation", "notifications"],
            }
        )
        self.__generate_stealth_options()

    def __generate_stealth_options(self) -> None:
        config = cast(StealthConfig, self._config)
        flags: Tuple[str, ...] = tuple()
        if not config.cdp_url:
            flags = tuple(DEFAULT_ARGS) + tuple(STEALTH_ARGS)

            if config.block_webrtc:
                flags += (
                    "--webrtc-ip-handling-policy=disable_non_proxied_udp",
                    "--force-webrtc-ip-handling-policy",  # Ensures the policy is enforced
                )
            if not config.allow_webgl:
                flags += (
                    "--disable-webgl",
                    "--disable-webgl-image-chromium",
                    "--disable-webgl2",
                )
            if config.hide_canvas:
                flags += ("--fingerprinting-canvas-image-data-noise",)

        super(StealthySessionMixin, self).__generate_options__(flags)

    @staticmethod
    def _detect_cloudflare(page_content: str) -> str | None:
        """
        Detect the type of Cloudflare challenge present in the provided page content.

        This function analyzes the given page content to identify whether a specific
        type of Cloudflare challenge is present. It checks for three predefined
        challenge types: non-interactive, managed, and interactive. If a challenge
        type is detected, it returns the corresponding type as a string. If no
        challenge type is detected, it returns None.

        Args:
            page_content (str): The content of the page to analyze for Cloudflare
                challenge types.

        Returns:
            str: A string representing the detected Cloudflare challenge type, if
                found. Returns None if no challenge matches.
        """
        challenge_types = (
            "non-interactive",
            "managed",
            "interactive",
        )
        for ctype in challenge_types:
            if f"cType: '{ctype}'" in page_content:
                return ctype

        # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe)
        selector = Selector(content=page_content)
        if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'):
            return "embedded"

        return None

    @classmethod
    def _challenge_cleared(cls, page_content: str, challenge_type: str) -> bool:
        """
        Check whether the Cloudflare challenge is no longer present in the page content.

        Args:
            page_content (str): The content of the page to analyze.
            challenge_type (str): The challenge type returned by `_detect_cloudflare`.

        Returns:
            bool: True if the challenge is gone from the page, False otherwise.
        """
        return challenge_type == "embedded" or cls._detect_cloudflare(page_content) is None