🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(shell): apply curl -A and pass -x proxies as a URL string (#490)
Suliman Abdulrazzaqbd90176c6602
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 779b02a..c86a3e4 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -235,6 +235,10 @@ class CurlParser:headers, cookies = _ParseHeaders(parsed_args.header)+ # `-A` only applies when `-H` didn't already set a user agent+ if parsed_args.user_agent and not any(key.lower() == "user-agent" for key in headers):+ headers["User-Agent"] = parsed_args.user_agent+if parsed_args.cookie:# We are focusing on the string format from DevTools.try:@@ -349,6 +353,9 @@ class CurlParser:return Nonerequest_args = request._asdict()+ if isinstance(request_args["proxy"], dict):+ # The fetchers take a single proxy URL, not a per-scheme dict+ request_args["proxy"] = request_args["proxy"].get("https") or request_args["proxy"].get("http")method = request_args.pop("method").strip().lower()if method in self._supported_methods:request_args["json"] = request_args.pop("json_data")diff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.pyindex 5b0de7d..6bf9e4d 100644--- a/tests/cli/test_shell_functionality.py+++ b/tests/cli/test_shell_functionality.py@@ -68,6 +68,36 @@ class TestCurlParser:assert "http://user:pass@proxy:8080" in request.proxy["http"]+ def test_curl_with_user_agent_option(self, parser):+ """Test that -A adds a User-Agent header unless -H already sets one"""+ request = parser.parse("curl https://example.com -A 'MyBot/1.0'")+ assert request.headers == {"User-Agent": "MyBot/1.0"}++ request = parser.parse("curl https://example.com --user-agent 'MyBot/1.0'")+ assert request.headers == {"User-Agent": "MyBot/1.0"}++ request = parser.parse("curl https://example.com -A 'MyBot/1.0' -H 'user-agent: Mozilla/5.0'")+ assert request.headers == {"user-agent": "Mozilla/5.0"}++ request = parser.parse("curl https://example.com -H 'Accept: text/html'")+ assert request.headers == {"Accept": "text/html"}++ @pytest.mark.parametrize(+ "curl_cmd, expected",+ [+ ("curl https://example.com -x proxy:8080", "http://proxy:8080"),+ ("curl https://example.com -x http://proxy:8080", "http://proxy:8080"),+ ("curl https://example.com -x http://proxy:8080 --proxy-user u:p", "http://u:p@proxy:8080"),+ ],+ )+ def test_curl2fetcher_proxy_is_a_url_string(self, parser, curl_cmd, expected):+ """Test that the proxy reaches the fetcher as a single URL string"""+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:+ _ = parser.convert2fetcher(curl_cmd)++ mock_get.assert_called_once()+ assert mock_get.call_args.kwargs["proxy"] == expected+def test_curl2fetcher(self, parser):"""Test converting curl to fetcher request"""with patch("scrapling.fetchers.Fetcher.get") as mock_get: