🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: Use shorter and more accurate naming for constants
Karim shoair16852cb1ac8b
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex e379882..51ce308 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -40,11 +40,7 @@ from scrapling.core._types import (Generator,AsyncGenerator,)-from scrapling.engines.constants import (- DEFAULT_STEALTH_FLAGS,- HARMFUL_DEFAULT_ARGS,- DEFAULT_FLAGS,-)+from scrapling.engines.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGSclass SyncSession:@@ -389,8 +385,8 @@ class BaseSessionMixin:# Dark color scheme bypasses the 'prefersLightColor' check in creepjsself._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}self._browser_options: Dict[str, Any] = {- "args": DEFAULT_FLAGS,- "ignore_default_args": HARMFUL_DEFAULT_ARGS,+ "args": DEFAULT_ARGS,+ "ignore_default_args": HARMFUL_ARGS,}if "__max_pages" in params:params["max_pages"] = params.pop("__max_pages")@@ -484,7 +480,7 @@ class StealthySessionMixin(BaseSessionMixin):config = cast(StealthConfig, self._config)flags: Tuple[str, ...] = tuple()if not config.cdp_url:- flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS+ flags = DEFAULT_ARGS + STEALTH_ARGSif config.block_webrtc:flags += (diff --git a/scrapling/engines/constants.py b/scrapling/engines/constants.pyindex f658416..4bc0014 100644--- a/scrapling/engines/constants.py+++ b/scrapling/engines/constants.py@@ -1,5 +1,5 @@# Disable loading these resources for speed-DEFAULT_DISABLED_RESOURCES = {+EXTRA_RESOURCES = {"font","image","media",@@ -12,7 +12,7 @@ DEFAULT_DISABLED_RESOURCES = {"stylesheet",}-HARMFUL_DEFAULT_ARGS = (+HARMFUL_ARGS = (# This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884"--enable-automation","--disable-popup-blocking",@@ -21,7 +21,7 @@ HARMFUL_DEFAULT_ARGS = ("--disable-extensions",)-DEFAULT_FLAGS = (+DEFAULT_ARGS = (# Speed up chromium browsers by default"--no-pings","--no-first-run",@@ -36,7 +36,7 @@ DEFAULT_FLAGS = ("--disable-search-engine-choice-screen",)-DEFAULT_STEALTH_FLAGS = (+STEALTH_ARGS = (# Explanation: https://peter.sh/experiments/chromium-command-line-switches/# Generally this will make the browser faster and less detectable# "--incognito",diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex e9e74ae..5c19427 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -12,7 +12,7 @@ from playwright.sync_api import Routefrom scrapling.core.utils import logfrom scrapling.core._types import Dict, Set, Tuple, Optional, Callable-from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES+from scrapling.engines.constants import EXTRA_RESOURCES__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"@@ -30,7 +30,7 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[:param blocked_domains: Set of domain names to block requests to.:return: A sync route handler function."""- disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()+ disabled_resources = EXTRA_RESOURCES if disable_resources else set()domains = blocked_domains or set()def handler(route: Route):@@ -57,7 +57,7 @@ def create_async_intercept_handler(disable_resources: bool, blocked_domains: Opt:param blocked_domains: Set of domain names to block requests to.:return: An async route handler function."""- disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()+ disabled_resources = EXTRA_RESOURCES if disable_resources else set()domains = blocked_domains or set()async def handler(route: async_Route):diff --git a/tests/fetchers/test_constants.py b/tests/fetchers/test_constants.pyindex 36662c6..91afd4c 100644--- a/tests/fetchers/test_constants.py+++ b/tests/fetchers/test_constants.py@@ -1,9 +1,4 @@-from scrapling.engines.constants import (- DEFAULT_DISABLED_RESOURCES,- DEFAULT_STEALTH_FLAGS,- HARMFUL_DEFAULT_ARGS,- DEFAULT_FLAGS,-)+from scrapling.engines.constants import EXTRA_RESOURCES, STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGSclass TestConstants:@@ -11,18 +6,18 @@ class TestConstants:def test_default_disabled_resources(self):"""Test default disabled resources"""- assert "image" in DEFAULT_DISABLED_RESOURCES- assert "font" in DEFAULT_DISABLED_RESOURCES- assert "stylesheet" in DEFAULT_DISABLED_RESOURCES- assert "media" in DEFAULT_DISABLED_RESOURCES+ assert "image" in EXTRA_RESOURCES+ assert "font" in EXTRA_RESOURCES+ assert "stylesheet" in EXTRA_RESOURCES+ assert "media" in EXTRA_RESOURCESdef test_harmful_default_args(self):"""Test harmful default arguments"""- assert "--enable-automation" in HARMFUL_DEFAULT_ARGS- assert "--disable-popup-blocking" in HARMFUL_DEFAULT_ARGS+ assert "--enable-automation" in HARMFUL_ARGS+ assert "--disable-popup-blocking" in HARMFUL_ARGSdef test_flags(self):"""Test default stealth flags"""- assert "--no-pings" in DEFAULT_FLAGS- # assert "--incognito" in DEFAULT_STEALTH_FLAGS- assert "--disable-blink-features=AutomationControlled" in DEFAULT_STEALTH_FLAGS+ assert "--no-pings" in DEFAULT_ARGS+ # assert "--incognito" in STEALTH_ARGS+ assert "--disable-blink-features=AutomationControlled" in STEALTH_ARGSdiff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex 9c08f7f..4637a63 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -8,7 +8,6 @@ from scrapling.engines.toolbelt.navigation import (create_async_intercept_handler,js_bypass_path,)-from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCESfrom scrapling.engines.toolbelt.fingerprints import (generate_convincing_referer,get_os_name,