🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(DynamicFetcher): Use persistent context by default for better stealth
Karim shoairdbf7aa7761a4
diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.pyindex b63b7f6..a2ee057 100644--- a/scrapling/engines/_browsers/_config_tools.py+++ b/scrapling/engines/_browsers/_config_tools.py@@ -56,16 +56,43 @@ def _set_flags(hide_canvas, disable_webgl):@lru_cache(2, typed=True)-def _launch_kwargs(headless, real_chrome, stealth, hide_canvas, disable_webgl) -> Tuple:+def _launch_kwargs(+ headless,+ proxy,+ locale,+ extra_headers,+ useragent,+ real_chrome,+ stealth,+ hide_canvas,+ disable_webgl,+) -> Tuple:"""Creates the arguments we will use while launching playwright's browser"""launch_kwargs = {"headless": headless,"ignore_default_args": HARMFUL_DEFAULT_ARGS,"channel": "chrome" if real_chrome else "chromium",+ "proxy": proxy or tuple(),+ "locale": locale,+ "color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs+ "device_scale_factor": 2,+ "extra_http_headers": extra_headers or tuple(),+ "user_agent": useragent or __default_useragent__,}if stealth:launch_kwargs.update(- {"args": _set_flags(hide_canvas, disable_webgl), "chromium_sandbox": True}+ {+ "args": _set_flags(hide_canvas, disable_webgl),+ "chromium_sandbox": True,+ "is_mobile": False,+ "has_touch": False,+ # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now+ "service_workers": "allow",+ "ignore_https_errors": True,+ "screen": {"width": 1920, "height": 1080},+ "viewport": {"width": 1920, "height": 1080},+ "permissions": ["geolocation", "notifications"],+ })return tuple(launch_kwargs.items())diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 213e1a6..1c5dd89 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -5,7 +5,6 @@ from playwright.sync_api import (Response as SyncPlaywrightResponse,sync_playwright,BrowserType,- Browser,BrowserContext,Playwright,Locator,@@ -14,7 +13,6 @@ from playwright.async_api import (async_playwright,Response as AsyncPlaywrightResponse,BrowserType as AsyncBrowserType,- Browser as AsyncBrowser,BrowserContext as AsyncBrowserContext,Playwright as AsyncPlaywright,Locator as AsyncLocator,@@ -177,7 +175,6 @@ class DynamicSession:self.wait_selector_state = config.wait_selector_stateself.playwright: Optional[Playwright] = None- self.browser: Optional[Union[BrowserType, Browser]] = Noneself.context: Optional[BrowserContext] = Noneself.page_pool = PagePool(self.max_pages)self._closed = False@@ -191,15 +188,25 @@ class DynamicSession:self.__initiate_browser_options__()def __initiate_browser_options__(self):+ # `launch_options` is used with persistent contextself.launch_options = dict(_launch_kwargs(self.headless,+ self.proxy,+ self.locale,+ tuple(self.extra_headers.items()) if self.extra_headers else tuple(),+ self.useragent,self.real_chrome,self.stealth,self.hide_canvas,self.disable_webgl,))+ self.launch_options["extra_http_headers"] = dict(+ self.launch_options["extra_http_headers"]+ )+ self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None+ # while `context_options` is left to be used when cdp mode is enabledself.context_options = dict(_context_kwargs(self.proxy,@@ -223,15 +230,17 @@ class DynamicSession:self.playwright = sync_context().start()- browser_launcher = getattr(+ browser_launcher: BrowserType = getattr(self.playwright, "chrome" if self.real_chrome else "chromium")if self.cdp_url:- self.browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)+ browser = browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)+ self.context = browser.new_context(**self.context_options)else:- self.browser = browser_launcher.launch(**self.launch_options)+ self.context = browser_launcher.launch_persistent_context(+ user_data_dir="", **self.launch_options+ )- self.context = self.browser.new_context(**self.context_options)if self.cookies:self.context.add_cookies(self.cookies)@@ -251,10 +260,6 @@ class DynamicSession:self.context.close()self.context = None- if self.browser:- self.browser.close()- self.browser = None-if self.playwright:self.playwright.stop()self.playwright = None@@ -459,7 +464,6 @@ class AsyncDynamicSession(DynamicSession):)self.playwright: Optional[AsyncPlaywright] = None- self.browser: Optional[Union[AsyncBrowserType, AsyncBrowser]] = Noneself.context: Optional[AsyncBrowserContext] = Noneself._lock = Lock()self.__enter__ = None@@ -478,15 +482,16 @@ class AsyncDynamicSession(DynamicSession):self.playwright, "chrome" if self.real_chrome else "chromium")if self.cdp_url:- self.browser = await browser_launcher.connect_over_cdp(- endpoint_url=self.cdp_url+ browser = await browser_launcher.connect_over_cdp(endpoint_url=self.cdp_url)+ self.context: AsyncBrowserContext = await browser.new_context(+ **self.context_options)else:- self.browser = await browser_launcher.launch(**self.launch_options)-- self.context: AsyncBrowserContext = await self.browser.new_context(- **self.context_options- )+ self.context: AsyncBrowserContext = (+ await browser_launcher.launch_persistent_context(+ user_data_dir="", **self.launch_options+ )+ )if self.cookies:await self.context.add_cookies(self.cookies)@@ -507,10 +512,6 @@ class AsyncDynamicSession(DynamicSession):await self.context.close()self.context = None- if self.browser:- await self.browser.close()- self.browser = None-if self.playwright:await self.playwright.stop()self.playwright = None