🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(browsers): add new feature to enable DNS-over-HTTP to prevent DNS leaks
Karim shoairbe28fe16ec15
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 9ca0dd4..3327dcb 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -309,6 +309,11 @@ def _common_browser_options(f):default=True,help="Run browser in headless mode (default: True)",),+ option(+ "--dns-over-https/--no-dns-over-https",+ default=False,+ help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)",+ ),option("--block-ads/--no-block-ads",default=False,@@ -503,6 +508,7 @@ def __build_browser_kwargs(real_chrome,proxy,parsed_headers,+ dns_over_https,block_ads,) -> Dict[str, Any]:"""Build shared kwargs dict for browser-based commands."""@@ -513,6 +519,7 @@ def __build_browser_kwargs("timeout": timeout,"locale": locale,"real_chrome": real_chrome,+ "dns_over_https": dns_over_https,"block_ads": block_ads,}if wait > 0:@@ -545,6 +552,7 @@ def fetch(proxy,extra_headers,ai_targeted,+ dns_over_https,block_ads,):"""Opens up a browser and fetch content using DynamicFetcher."""@@ -560,6 +568,7 @@ def fetch(real_chrome,proxy,parsed_headers,+ dns_over_https,block_ads,)from scrapling.fetchers import DynamicFetcher@@ -606,6 +615,7 @@ def stealthy_fetch(allow_webgl,hide_canvas,ai_targeted,+ dns_over_https,block_ads,):"""Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""@@ -621,6 +631,7 @@ def stealthy_fetch(real_chrome,proxy,parsed_headers,+ dns_over_https,block_ads,)kwargs.update(diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex afa482d..b2340fb 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -65,6 +65,7 @@ _FETCH_PARAMS = {"retry_delay": int | float,"capture_xhr": str | None,"executable_path": Optional[str],+ "dns_over_https": bool,}_STEALTHY_FETCH_PARAMS = {@@ -97,6 +98,7 @@ _STEALTHY_FETCH_PARAMS = {"retry_delay": int | float,"capture_xhr": str | None,"executable_path": Optional[str],+ "dns_over_https": bool,"allow_webgl": bool,"hide_canvas": bool,"block_webrtc": bool,diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 2d42b08..c9d390c 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -455,6 +455,13 @@ class BaseSessionMixin:if config.extra_flags or extra_flags:flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))+ if config.dns_over_https:+ doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"+ if isinstance(flags, list):+ flags.append(doh_flag)+ else:+ flags = list(flags) + [doh_flag]+self._browser_options.update({"args": flags,diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 932b2e2..5030480 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -93,6 +93,7 @@ class PlaywrightSession(TypedDict, total=False):retry_delay: int | floatcapture_xhr: str | Noneexecutable_path: Optional[str]+ dns_over_https: boolclass PlaywrightFetchParams(TypedDict, total=False):diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 8d47061..f7bb34e 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -90,6 +90,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):retry_delay: Seconds = 1capture_xhr: str | None = Noneexecutable_path: Optional[str] = None+ dns_over_https: bool = Falsedef __post_init__(self): # pragma: no cover"""Custom validation after msgspec validation"""diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.pyindex 0c06aa1..f7795b7 100644--- a/scrapling/fetchers/chrome.py+++ b/scrapling/fetchers/chrome.py@@ -16,6 +16,7 @@ class DynamicFetcher(BaseFetcher)::param disable_resources: Drop requests for unnecessary resources for a speed boost.:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.@@ -57,6 +58,7 @@ class DynamicFetcher(BaseFetcher)::param disable_resources: Drop requests for unnecessary resources for a speed boost.:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.pyindex 0c7134c..70ce937 100644--- a/scrapling/fetchers/stealth_chrome.py+++ b/scrapling/fetchers/stealth_chrome.py@@ -21,6 +21,7 @@ class StealthyFetcher(BaseFetcher):Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.@@ -71,6 +72,7 @@ class StealthyFetcher(BaseFetcher):Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.