Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(DynamicSession): Improve normal fetches speed

Karim shoair6eb580a6d0bb
diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py
index 7d29a16..477bb21 100644
--- a/scrapling/engines/__init__.py
+++ b/scrapling/engines/__init__.py
@@ -1,4 +1,4 @@
-from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
+from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS, DEFAULT_FLAGS
from .static import FetcherSession, FetcherClient, AsyncFetcherClient
from ._browsers import (
DynamicSession,
diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py
index 46d2ad6..7f60efc 100644
--- a/scrapling/engines/_browsers/_config_tools.py
+++ b/scrapling/engines/_browsers/_config_tools.py
@@ -1,7 +1,11 @@
from functools import lru_cache
from scrapling.core._types import Tuple
-from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, HARMFUL_DEFAULT_ARGS
+from scrapling.engines.constants import (
+ DEFAULT_STEALTH_FLAGS,
+ HARMFUL_DEFAULT_ARGS,
+ DEFAULT_FLAGS,
+)
from scrapling.engines.toolbelt import js_bypass_path, generate_headers
__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
@@ -69,20 +73,21 @@ def _launch_kwargs(
) -> Tuple:
"""Creates the arguments we will use while launching playwright's browser"""
launch_kwargs = {
- "headless": headless,
- "ignore_default_args": HARMFUL_DEFAULT_ARGS,
- "channel": "chrome" if real_chrome else "chromium",
- "proxy": proxy or tuple(),
"locale": locale,
+ "headless": headless,
+ "args": DEFAULT_FLAGS,
"color_scheme": "dark", # Bypasses the 'prefersLightColor' check in creepjs
+ "proxy": proxy or tuple(),
"device_scale_factor": 2,
+ "ignore_default_args": HARMFUL_DEFAULT_ARGS,
+ "channel": "chrome" if real_chrome else "chromium",
"extra_http_headers": extra_headers or tuple(),
"user_agent": useragent or __default_useragent__,
}
if stealth:
launch_kwargs.update(
{
- "args": _set_flags(hide_canvas, disable_webgl),
+ "args": DEFAULT_FLAGS + _set_flags(hide_canvas, disable_webgl),
"chromium_sandbox": True,
"is_mobile": False,
"has_touch": False,
diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.py
index 2a84a2d..c7bdb1b 100644
--- a/scrapling/engines/constants.py
+++ b/scrapling/engines/constants.py
@@ -21,22 +21,32 @@ HARMFUL_DEFAULT_ARGS = (
# '--disable-extensions',
)
+DEFAULT_FLAGS = (
+ # Speed up chromium browsers by default
+ "--no-pings",
+ "--no-first-run",
+ "--disable-infobars",
+ "--disable-breakpad",
+ "--no-service-autorun",
+ "--homepage=about:blank",
+ "--password-store=basic",
+ "--no-default-browser-check",
+ "--disable-session-crashed-bubble",
+ "--disable-search-engine-choice-screen",
+)
+
DEFAULT_STEALTH_FLAGS = (
# Explanation: https://peter.sh/experiments/chromium-command-line-switches/
# Generally this will make the browser faster and less detectable
- "--no-pings",
"--incognito",
"--test-type",
"--lang=en-US",
"--mute-audio",
- "--no-first-run",
"--disable-sync",
"--hide-scrollbars",
"--disable-logging",
"--start-maximized", # For headless check bypass
"--enable-async-dns",
- "--disable-breakpad",
- "--disable-infobars",
"--accept-lang=en-US",
"--use-mock-keychain",
"--disable-translate",
@@ -48,7 +58,6 @@ DEFAULT_STEALTH_FLAGS = (
"--enable-tcp-fast-open",
"--enable-web-bluetooth",
"--disable-hang-monitor",
- "--password-store=basic",
"--disable-cloud-import",
"--disable-default-apps",
"--disable-print-preview",
@@ -62,17 +71,15 @@ DEFAULT_STEALTH_FLAGS = (
"--disable-prompt-on-repost",
"--force-color-profile=srgb",
"--font-render-hinting=none",
- "--no-default-browser-check",
"--aggressive-cache-discard",
"--disable-component-update",
"--disable-cookie-encryption",
"--disable-domain-reliability",
"--disable-threaded-animation",
"--disable-threaded-scrolling",
- # '--disable-reading-from-canvas', # For Firefox
+ # '--disable-reading-from-canvas', # For Firefox
"--enable-simple-cache-backend",
"--disable-background-networking",
- "--disable-session-crashed-bubble",
"--enable-surface-synchronization",
"--disable-image-animation-resync",
"--disable-renderer-backgrounding",