Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(static): exclude block_ads from HTTP request args

voidborne-d76ba28efaafb
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index fae9f83..033e20e 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -149,6 +149,7 @@ class _ConfigurationLogic(ABC):
# Browser session params (ignored by HTTP sessions)
"extra_headers",
"google_search",
+ "block_ads",
}
for k, v in method_kwargs.items():
if k not in skip_keys and v is not None:
diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.py
new file mode 100644
index 0000000..6b1e101
--- /dev/null
+++ b/tests/fetchers/test_merge_request_args.py
@@ -0,0 +1,44 @@
+"""Tests for _merge_request_args to ensure browser-only kwargs are excluded.
+
+Regression tests for https://github.com/D4Vinci/Scrapling/issues/247
+"""
+
+import pytest
+
+from scrapling.engines.static import FetcherClient
+
+
+class TestMergeRequestArgsSkipsBrowserParams:
+ """Verify that browser-only keyword arguments are stripped before
+ the request dict is forwarded to curl_cffi's Session.request()."""
+
+ def _build_args(self, **extra_kwargs):
+ """Helper: instantiate a FetcherClient and call _merge_request_args."""
+ client = FetcherClient()
+ return client._merge_request_args(url="https://example.com", **extra_kwargs)
+
+ def test_block_ads_excluded(self):
+ """block_ads is a browser-engine param and must not leak into the
+ HTTP request dict (fixes #247)."""
+ args = self._build_args(block_ads=True)
+ assert "block_ads" not in args
+
+ def test_google_search_excluded(self):
+ """google_search is a browser-engine param and should be stripped."""
+ args = self._build_args(google_search=True)
+ assert "google_search" not in args
+
+ def test_extra_headers_excluded(self):
+ """extra_headers is a browser-engine param and should be stripped."""
+ args = self._build_args(extra_headers={"X-Custom": "val"})
+ assert "extra_headers" not in args
+
+ def test_url_present(self):
+ """The url must always be present in the output dict."""
+ args = self._build_args()
+ assert args["url"] == "https://example.com"
+
+ def test_valid_kwargs_passed_through(self):
+ """Arbitrary curl_cffi-compatible kwargs should survive."""
+ args = self._build_args(cookies={"session": "abc"})
+ assert args.get("cookies") == {"session": "abc"}