Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

Percent-encode proxy credentials in the HTTP crawler proxy URL

omarbounawarapy2de19c8b0a16
diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py
index 6d0fb476..e46e5068 100644
--- a/crawl4ai/async_crawler_strategy.py
+++ b/crawl4ai/async_crawler_strategy.py
@@ -28,7 +28,7 @@ import aiofiles
import aiohttp
import chardet
from aiohttp.client import ClientTimeout
-from urllib.parse import urlparse
+from urllib.parse import quote, urlparse
from types import MappingProxyType
import contextlib
from functools import partial
@@ -2642,6 +2642,8 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
if username and password:
# Insert credentials into URL: http://user:pass@host:port
+ username = quote(username, safe='')
+ password = quote(password, safe='')
if '://' in server:
protocol, rest = server.split('://', 1)
return f"{protocol}://{username}:{password}@{rest}"
diff --git a/tests/proxy/test_http_proxy_url_encoding.py b/tests/proxy/test_http_proxy_url_encoding.py
new file mode 100644
index 00000000..f4bcbbd5
--- /dev/null
+++ b/tests/proxy/test_http_proxy_url_encoding.py
@@ -0,0 +1,16 @@
+import pytest
+from yarl import URL
+
+from crawl4ai.async_configs import ProxyConfig
+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
+
+
[email protected]("password", ["plainpass", "p@ss", "p#ss", "pa/ss", "p?ss", "p%41ss"])
+def test_proxy_url_roundtrips_credentials(password):
+ strategy = AsyncHTTPCrawlerStrategy()
+ config = ProxyConfig(server="http://host:8080", username="us:er", password=password)
+ url = URL(strategy._format_proxy_url(config))
+ assert url.host == "host"
+ assert url.port == 8080
+ assert url.user == "us:er"
+ assert url.password == password