Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(DynamicFetcher): Use persistent context by default for better stealth

Karim shoairdbf7aa7761a4
diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py
index b63b7f6..a2ee057 100644
--- a/scrapling/engines/_browsers/_config_tools.py
+++ b/scrapling/engines/_browsers/_config_tools.py
@@ -56,16 +56,43 @@ def _set_flags(hide_canvas, disable_webgl):
@lru_cache(2, typed=True)
-def _launch_kwargs(headless, real_chrome, stealth, hide_canvas, disable_webgl) -> Tuple:
+def _launch_kwargs(
+ headless,
+ proxy,
+ locale,
+ extra_headers,
+ useragent,
+ real_chrome,
+ stealth,
+ hide_canvas,
+ disable_webgl,
+) -> Tuple:
"""Creates the arguments we will use while launching playwright's browser"""
launch_kwargs = {
"headless": headless,
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
"channel": "chrome" if real_chrome else "chromium",
+ "proxy": proxy or tuple(),
+ "locale": locale,
+ "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
+ "device_scale_factor": 2,
+ "extra_http_headers": extra_headers or tuple(),
+ "user_agent": useragent or __default_useragent__,
}
if stealth:
launch_kwargs.update(
- {"args": _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True}
+ {
+ "args": _set_flags(hide_canvas, disable_webgl),
+ "chromium_sandbox": True,
+ "is_mobile": False,
+ "has_touch": False,
+ # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
+ "service_workers": "allow",
+ "ignore_https_errors": True,
+ "screen": {"width": 1920, "height": 1080},
+ "viewport": {"width": 1920, "height": 1080},
+ "permissions": ["geolocation", "notifications"],
+ }
)
return tuple(launch_kwargs.items())
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 213e1a6..1c5dd89 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -5,7 +5,6 @@ from playwright.sync_api import (
Response as SyncPlaywrightResponse,
sync_playwright,
BrowserType,
- Browser,
BrowserContext,
Playwright,
Locator,
@@ -14,7 +13,6 @@ from playwright.async_api import (
async_playwright,
Response as AsyncPlaywrightResponse,
BrowserType as AsyncBrowserType,
- Browser as AsyncBrowser,
BrowserContext as AsyncBrowserContext,
Playwright as AsyncPlaywright,
Locator as AsyncLocator,
@@ -177,7 +175,6 @@ class DynamicSession:
self.wait_selector_state = config.wait_selector_state
self.playwright: Optional[Playwright] = None
- self.browser: Optional[Union[BrowserType, Browser]] = None
self.context: Optional[BrowserContext] = None
self.page_pool = PagePool(self.max_pages)
self._closed = False
@@ -191,15 +188,25 @@ class DynamicSession:
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
+ # `launch_options` is used with persistent context
self.launch_options = dict(
_launch_kwargs(
self.headless,
+ self.proxy,
+ self.locale,
+ tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
+ self.useragent,
self.real_chrome,
self.stealth,
self.hide_canvas,
self.disable_webgl,
)
)
+ self.launch_options["extra_http_headers"] = dict(
+ self.launch_options["extra_http_headers"]
+ )
+ self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
+ # while `context_options` is left to be used when cdp mode is enabled
self.context_options = dict(
_context_kwargs(
self.proxy,
@@ -223,15 +230,17 @@ class DynamicSession:
self.playwright = sync_context().start()
- browser_launcher = getattr(
+ browser_launcher: BrowserType = getattr(
self.playwright, "chrome" if self.real_chrome else "chromium"
)
if self.cdp_url:
- self.browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+ browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+ self.context = browser.new_context(**self.context_options)
else:
- self.browser = browser_launcher.launch(**self.launch_options)
+ self.context = browser_launcher.launch_persistent_context(
+ user_data_dir="", **self.launch_options
+ )
- self.context = self.browser.new_context(**self.context_options)
if self.cookies:
self.context.add_cookies(self.cookies)
@@ -251,10 +260,6 @@ class DynamicSession:
self.context.close()
self.context = None
- if self.browser:
- self.browser.close()
- self.browser = None
-
if self.playwright:
self.playwright.stop()
self.playwright = None
@@ -459,7 +464,6 @@ class AsyncDynamicSession(DynamicSession):
)
self.playwright: Optional[AsyncPlaywright] = None
- self.browser: Optional[Union[AsyncBrowserType, AsyncBrowser]] = None
self.context: Optional[AsyncBrowserContext] = None
self._lock = Lock()
self.__enter__ = None
@@ -478,15 +482,16 @@ class AsyncDynamicSession(DynamicSession):
self.playwright, "chrome" if self.real_chrome else "chromium"
)
if self.cdp_url:
- self.browser = await browser_launcher.connect_over_cdp(
- endpoint_url=self.cdp_url
+ browser = await browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)
+ self.context: AsyncBrowserContext = await browser.new_context(
+ **self.context_options
)
else:
- self.browser = await browser_launcher.launch(**self.launch_options)
-
- self.context: AsyncBrowserContext = await self.browser.new_context(
- **self.context_options
- )
+ self.context: AsyncBrowserContext = (
+ await browser_launcher.launch_persistent_context(
+ user_data_dir="", **self.launch_options
+ )
+ )
if self.cookies:
await self.context.add_cookies(self.cookies)
@@ -507,10 +512,6 @@ class AsyncDynamicSession(DynamicSession):
await self.context.close()
self.context = None
- if self.browser:
- await self.browser.close()
- self.browser = None
-
if self.playwright:
await self.playwright.stop()
self.playwright = None