🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(DynamicSession): Improve normal fetches speed
Karim shoair6eb580a6d0bb
diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.pyindex 7d29a16..477bb21 100644--- a/scrapling/engines/__init__.py+++ b/scrapling/engines/__init__.py@@ -1,4 +1,4 @@-from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS+from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGSfrom .static import FetcherSession, FetcherClient, AsyncFetcherClientfrom ._browsers import (DynamicSession,diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.pyindex 46d2ad6..7f60efc 100644--- a/scrapling/engines/_browsers/_config_tools.py+++ b/scrapling/engines/_browsers/_config_tools.py@@ -1,7 +1,11 @@from functools import lru_cachefrom scrapling.core._types import Tuple-from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, HARMFUL_DEFAULT_ARGS+from scrapling.engines.constants import (+ DEFAULT_STEALTH_FLAGS,+ HARMFUL_DEFAULT_ARGS,+ DEFAULT_FLAGS,+)from scrapling.engines.toolbelt import js_bypass_path, generate_headers__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")@@ -69,20 +73,21 @@ def _launch_kwargs() -> Tuple:"""Creates the arguments we will use while launching playwright's browser"""launch_kwargs = {- "headless": headless,- "ignore_default_args": HARMFUL_DEFAULT_ARGS,- "channel": "chrome" if real_chrome else "chromium",- "proxy": proxy or tuple(),"locale": locale,+ "headless": headless,+ "args": DEFAULT_FLAGS,"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs+ "proxy": proxy or tuple(),"device_scale_factor": 2,+ "ignore_default_args": HARMFUL_DEFAULT_ARGS,+ "channel": "chrome" if real_chrome else "chromium","extra_http_headers": extra_headers or tuple(),"user_agent": useragent or __default_useragent__,}if stealth:launch_kwargs.update({- "args": _set_flags(hide_canvas, disable_webgl),+ "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl),"chromium_sandbox": True,"is_mobile": False,"has_touch": False,diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.pyindex 2a84a2d..c7bdb1b 100644--- a/scrapling/engines/constants.py+++ b/scrapling/engines/constants.py@@ -21,22 +21,32 @@ HARMFUL_DEFAULT_ARGS = (# '--disable-extensions',)+DEFAULT_FLAGS = (+ # Speed up chromium browsers by default+ "--no-pings",+ "--no-first-run",+ "--disable-infobars",+ "--disable-breakpad",+ "--no-service-autorun",+ "--homepage=about:blank",+ "--password-store=basic",+ "--no-default-browser-check",+ "--disable-session-crashed-bubble",+ "--disable-search-engine-choice-screen",+)+DEFAULT_STEALTH_FLAGS = (# Explanation: https://peter.sh/experiments/chromium-command-line-switches/# Generally this will make the browser faster and less detectable- "--no-pings","--incognito","--test-type","--lang=en-US","--mute-audio",- "--no-first-run","--disable-sync","--hide-scrollbars","--disable-logging","--start-maximized", # For headless check bypass"--enable-async-dns",- "--disable-breakpad",- "--disable-infobars","--accept-lang=en-US","--use-mock-keychain","--disable-translate",@@ -48,7 +58,6 @@ DEFAULT_STEALTH_FLAGS = ("--enable-tcp-fast-open","--enable-web-bluetooth","--disable-hang-monitor",- "--password-store=basic","--disable-cloud-import","--disable-default-apps","--disable-print-preview",@@ -62,17 +71,15 @@ DEFAULT_STEALTH_FLAGS = ("--disable-prompt-on-repost","--force-color-profile=srgb","--font-render-hinting=none",- "--no-default-browser-check","--aggressive-cache-discard","--disable-component-update","--disable-cookie-encryption","--disable-domain-reliability","--disable-threaded-animation","--disable-threaded-scrolling",- # '--disable-reading-from-canvas', # For Firefox+ # '--disable-reading-from-canvas', # For Firefox"--enable-simple-cache-backend","--disable-background-networking",- "--disable-session-crashed-bubble","--enable-surface-synchronization","--disable-image-animation-resync","--disable-renderer-backgrounding",