Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(shell): apply curl -A and pass -x proxies as a URL string (#490)

Suliman Abdulrazzaqbd90176c6602
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 779b02a..c86a3e4 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -235,6 +235,10 @@ class CurlParser:
headers, cookies = _ParseHeaders(parsed_args.header)
+ # `-A` only applies when `-H` didn't already set a user agent
+ if parsed_args.user_agent and not any(key.lower() == "user-agent" for key in headers):
+ headers["User-Agent"] = parsed_args.user_agent
+
if parsed_args.cookie:
# We are focusing on the string format from DevTools.
try:
@@ -349,6 +353,9 @@ class CurlParser:
return None
request_args = request._asdict()
+ if isinstance(request_args["proxy"], dict):
+ # The fetchers take a single proxy URL, not a per-scheme dict
+ request_args["proxy"] = request_args["proxy"].get("https") or request_args["proxy"].get("http")
method = request_args.pop("method").strip().lower()
if method in self._supported_methods:
request_args["json"] = request_args.pop("json_data")
diff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.py
index 5b0de7d..6bf9e4d 100644
--- a/tests/cli/test_shell_functionality.py
+++ b/tests/cli/test_shell_functionality.py
@@ -68,6 +68,36 @@ class TestCurlParser:
assert "http://user:pass@proxy:8080" in request.proxy["http"]
+ def test_curl_with_user_agent_option(self, parser):
+ """Test that -A adds a User-Agent header unless -H already sets one"""
+ request = parser.parse("curl https://example.com -A 'MyBot/1.0'")
+ assert request.headers == {"User-Agent": "MyBot/1.0"}
+
+ request = parser.parse("curl https://example.com --user-agent 'MyBot/1.0'")
+ assert request.headers == {"User-Agent": "MyBot/1.0"}
+
+ request = parser.parse("curl https://example.com -A 'MyBot/1.0' -H 'user-agent: Mozilla/5.0'")
+ assert request.headers == {"user-agent": "Mozilla/5.0"}
+
+ request = parser.parse("curl https://example.com -H 'Accept: text/html'")
+ assert request.headers == {"Accept": "text/html"}
+
+ @pytest.mark.parametrize(
+ "curl_cmd, expected",
+ [
+ ("curl https://example.com -x proxy:8080", "http://proxy:8080"),
+ ("curl https://example.com -x http://proxy:8080", "http://proxy:8080"),
+ ("curl https://example.com -x http://proxy:8080 --proxy-user u:p", "http://u:p@proxy:8080"),
+ ],
+ )
+ def test_curl2fetcher_proxy_is_a_url_string(self, parser, curl_cmd, expected):
+ """Test that the proxy reaches the fetcher as a single URL string"""
+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:
+ _ = parser.convert2fetcher(curl_cmd)
+
+ mock_get.assert_called_once()
+ assert mock_get.call_args.kwargs["proxy"] == expected
+
def test_curl2fetcher(self, parser):
"""Test converting curl to fetcher request"""
with patch("scrapling.fetchers.Fetcher.get") as mock_get: