Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(StealthyFetcher): Remove virtual mode and use persistent context

Karim shoaird1aa0be6e4b1
diff --git a/scrapling/cli.py b/scrapling/cli.py
index ee59a72..e5e91d5 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -774,7 +774,7 @@ def stealthy_fetch(
:param url: Target url.
:param output_file: Output file path (.md for Markdown, .html for HTML).
- :param headless: Run the browser in headless/hidden, virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden, or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
:param disable_resources: Drop requests of unnecessary resources for a speed boost.
:param block_webrtc: Blocks WebRTC entirely.
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 07104f0..0231d86 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -410,7 +410,7 @@ class ScraplingMCPServer:
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
- headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
+ headless: Union[bool] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
@@ -443,7 +443,7 @@ class ScraplingMCPServer:
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
- :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
@@ -510,7 +510,7 @@ class ScraplingMCPServer:
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
- headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
+ headless: Union[bool] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
@@ -543,7 +543,7 @@ class ScraplingMCPServer:
- Text will return the text content of the page.
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
- :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index b886c97..7fbc839 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -2,12 +2,11 @@ from time import time, sleep
from re import compile as re_compile
from asyncio import sleep as asyncio_sleep, Lock
-from camoufox import AsyncNewBrowser, NewBrowser, DefaultAddons
+from camoufox import DefaultAddons
+from camoufox.utils import launch_options as generate_launch_options
from playwright.sync_api import (
Response as SyncPlaywrightResponse,
sync_playwright,
- BrowserType,
- Browser,
BrowserContext,
Playwright,
Locator,
@@ -16,8 +15,6 @@ from playwright.sync_api import (
from playwright.async_api import (
async_playwright,
Response as AsyncPlaywrightResponse,
- BrowserType as AsyncBrowserType,
- Browser as AsyncBrowser,
BrowserContext as AsyncBrowserContext,
Playwright as AsyncPlaywright,
Locator as AsyncLocator,
@@ -32,7 +29,6 @@ from scrapling.core._types import (
Optional,
Union,
Callable,
- Literal,
List,
SelectorWaitStates,
)
@@ -82,14 +78,13 @@ class StealthySession:
"page_pool",
"_closed",
"launch_options",
- "context_options",
"_headers_keys",
)
def __init__(
self,
max_pages: int = 1,
- headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
+ headless: Union[bool] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
@@ -115,7 +110,7 @@ class StealthySession:
):
"""A Browser session manager with page pooling
- :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
@@ -199,7 +194,6 @@ class StealthySession:
self.additional_arguments = config.additional_arguments
self.playwright: Optional[Playwright] = None
- self.browser: Optional[Union[BrowserType, Browser]] = None
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
@@ -214,28 +208,31 @@ class StealthySession:
def __initiate_browser_options__(self):
"""Initiate browser options."""
- self.launch_options = {
- "geoip": self.geoip,
- "proxy": dict(self.proxy) if self.proxy else self.proxy,
- "enable_cache": True,
- "addons": self.addons,
- "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
- "headless": self.headless,
- "humanize": True if self.solve_cloudflare else self.humanize,
- "i_know_what_im_doing": True, # To turn warnings off with the user configurations
- "allow_webgl": self.allow_webgl,
- "block_webrtc": self.block_webrtc,
- "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
- "os": None if self.os_randomize else get_os_name(),
- **self.additional_arguments,
- }
- self.context_options = {}
+ self.launch_options = generate_launch_options(
+ **{
+ "geoip": self.geoip,
+ "proxy": dict(self.proxy) if self.proxy else self.proxy,
+ "enable_cache": True,
+ "addons": self.addons,
+ "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],
+ "headless": self.headless,
+ "humanize": True if self.solve_cloudflare else self.humanize,
+ "i_know_what_im_doing": True, # To turn warnings off with the user configurations
+ "allow_webgl": self.allow_webgl,
+ "block_webrtc": self.block_webrtc,
+ "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.
+ "os": None if self.os_randomize else get_os_name(),
+ "user_data_dir": "",
+ **self.additional_arguments,
+ }
+ )
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
- self.browser = NewBrowser(self.playwright, **self.launch_options)
- self.context = self.browser.new_context(**self.context_options)
+ self.context = self.playwright.firefox.launch_persistent_context(
+ **self.launch_options
+ )
if self.cookies:
self.context.add_cookies(self.cookies)
@@ -255,10 +252,6 @@ class StealthySession:
self.context.close()
self.context = None
- if self.browser:
- self.browser.close()
- self.browser = None
-
if self.playwright:
self.playwright.stop()
self.playwright = None
@@ -468,7 +461,7 @@ class AsyncStealthySession(StealthySession):
def __init__(
self,
max_pages: int = 1,
- headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
+ headless: Union[bool] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
@@ -494,7 +487,7 @@ class AsyncStealthySession(StealthySession):
):
"""A Browser session manager with page pooling
- :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
@@ -550,7 +543,6 @@ class AsyncStealthySession(StealthySession):
additional_arguments,
)
self.playwright: Optional[AsyncPlaywright] = None
- self.browser: Optional[Union[AsyncBrowserType, AsyncBrowser]] = None
self.context: Optional[AsyncBrowserContext] = None
self._lock = Lock()
self.__enter__ = None
@@ -559,9 +551,10 @@ class AsyncStealthySession(StealthySession):
async def __create__(self):
"""Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start()
- self.browser = await AsyncNewBrowser(self.playwright, **self.launch_options)
- self.context: AsyncBrowserContext = await self.browser.new_context(
- **self.context_options
+ self.context: AsyncBrowserContext = (
+ await self.playwright.firefox.launch_persistent_context(
+ **self.launch_options
+ )
)
if self.cookies:
await self.context.add_cookies(self.cookies)
@@ -582,10 +575,6 @@ class AsyncStealthySession(StealthySession):
await self.context.close()
self.context = None
- if self.browser:
- await self.browser.close()
- self.browser = None
-
if self.playwright:
await self.playwright.stop()
self.playwright = None
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index e0409f5..e2a3024 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -82,7 +82,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
max_pages: int = 1
- headless: Union[bool, Literal["virtual"]] = True # noqa: F821
+ headless: Union[bool] = True # noqa: F821
block_images: bool = False
disable_resources: bool = False
block_webrtc: bool = False
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index e628a2c..3b7d7c9 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -52,7 +52,7 @@ class StealthyFetcher(BaseFetcher):
def fetch(
cls,
url: str,
- headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
+ headless: Union[bool] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
@@ -80,7 +80,7 @@ class StealthyFetcher(BaseFetcher):
Opens up a browser and do your request based on your chosen options below.
:param url: Target url.
- :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
@@ -148,7 +148,7 @@ class StealthyFetcher(BaseFetcher):
async def async_fetch(
cls,
url: str,
- headless: Union[bool, Literal["virtual"]] = True, # noqa: F821
+ headless: Union[bool] = True, # noqa: F821
block_images: bool = False,
disable_resources: bool = False,
block_webrtc: bool = False,
@@ -176,7 +176,7 @@ class StealthyFetcher(BaseFetcher):
Opens up a browser and do your request based on your chosen options below.
:param url: Target url.
- :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param block_images: Prevent the loading of images through Firefox preferences.
This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.