Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(browsers): add new feature to enable DNS-over-HTTP to prevent DNS leaks

Karim shoairbe28fe16ec15
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 9ca0dd4..3327dcb 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -309,6 +309,11 @@ def _common_browser_options(f):
default=True,
help="Run browser in headless mode (default: True)",
),
+ option(
+ "--dns-over-https/--no-dns-over-https",
+ default=False,
+ help="Route DNS through Cloudflare's DoH to prevent DNS leaks when using proxies (default: False)",
+ ),
option(
"--block-ads/--no-block-ads",
default=False,
@@ -503,6 +508,7 @@ def __build_browser_kwargs(
real_chrome,
proxy,
parsed_headers,
+ dns_over_https,
block_ads,
) -> Dict[str, Any]:
"""Build shared kwargs dict for browser-based commands."""
@@ -513,6 +519,7 @@ def __build_browser_kwargs(
"timeout": timeout,
"locale": locale,
"real_chrome": real_chrome,
+ "dns_over_https": dns_over_https,
"block_ads": block_ads,
}
if wait > 0:
@@ -545,6 +552,7 @@ def fetch(
proxy,
extra_headers,
ai_targeted,
+ dns_over_https,
block_ads,
):
"""Opens up a browser and fetch content using DynamicFetcher."""
@@ -560,6 +568,7 @@ def fetch(
real_chrome,
proxy,
parsed_headers,
+ dns_over_https,
block_ads,
)
from scrapling.fetchers import DynamicFetcher
@@ -606,6 +615,7 @@ def stealthy_fetch(
allow_webgl,
hide_canvas,
ai_targeted,
+ dns_over_https,
block_ads,
):
"""Opens up a browser with advanced stealth features and fetch content using StealthyFetcher."""
@@ -621,6 +631,7 @@ def stealthy_fetch(
real_chrome,
proxy,
parsed_headers,
+ dns_over_https,
block_ads,
)
kwargs.update(
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index afa482d..b2340fb 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -65,6 +65,7 @@ _FETCH_PARAMS = {
"retry_delay": int | float,
"capture_xhr": str | None,
"executable_path": Optional[str],
+ "dns_over_https": bool,
}
_STEALTHY_FETCH_PARAMS = {
@@ -97,6 +98,7 @@ _STEALTHY_FETCH_PARAMS = {
"retry_delay": int | float,
"capture_xhr": str | None,
"executable_path": Optional[str],
+ "dns_over_https": bool,
"allow_webgl": bool,
"hide_canvas": bool,
"block_webrtc": bool,
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 2d42b08..c9d390c 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -455,6 +455,13 @@ class BaseSessionMixin:
if config.extra_flags or extra_flags:
flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
+ if config.dns_over_https:
+ doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
+ if isinstance(flags, list):
+ flags.append(doh_flag)
+ else:
+ flags = list(flags) + [doh_flag]
+
self._browser_options.update(
{
"args": flags,
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 932b2e2..5030480 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -93,6 +93,7 @@ class PlaywrightSession(TypedDict, total=False):
retry_delay: int | float
capture_xhr: str | None
executable_path: Optional[str]
+ dns_over_https: bool
class PlaywrightFetchParams(TypedDict, total=False):
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 8d47061..f7bb34e 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -90,6 +90,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
retry_delay: Seconds = 1
capture_xhr: str | None = None
executable_path: Optional[str] = None
+ dns_over_https: bool = False
def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation"""
diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py
index 0c06aa1..f7795b7 100644
--- a/scrapling/fetchers/chrome.py
+++ b/scrapling/fetchers/chrome.py
@@ -16,6 +16,7 @@ class DynamicFetcher(BaseFetcher):
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -57,6 +58,7 @@ class DynamicFetcher(BaseFetcher):
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py
index 0c7134c..70ce937 100644
--- a/scrapling/fetchers/stealth_chrome.py
+++ b/scrapling/fetchers/stealth_chrome.py
@@ -21,6 +21,7 @@ class StealthyFetcher(BaseFetcher):
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -71,6 +72,7 @@ class StealthyFetcher(BaseFetcher):
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
+ :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.