Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
Percent-encode proxy credentials in the HTTP crawler proxy URL
omarbounawarapy2de19c8b0a16
diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.pyindex 6d0fb476..e46e5068 100644--- a/crawl4ai/async_crawler_strategy.py+++ b/crawl4ai/async_crawler_strategy.py@@ -28,7 +28,7 @@ import aiofilesimport aiohttpimport chardetfrom aiohttp.client import ClientTimeout-from urllib.parse import urlparse+from urllib.parse import quote, urlparsefrom types import MappingProxyTypeimport contextlibfrom functools import partial@@ -2642,6 +2642,8 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):if username and password:# Insert credentials into URL: http://user:pass@host:port+ username = quote(username, safe='')+ password = quote(password, safe='')if '://' in server:protocol, rest = server.split('://', 1)return f"{protocol}://{username}:{password}@{rest}"diff --git a/tests/proxy/test_http_proxy_url_encoding.py b/tests/proxy/test_http_proxy_url_encoding.pynew file mode 100644index 00000000..f4bcbbd5--- /dev/null+++ b/tests/proxy/test_http_proxy_url_encoding.py@@ -0,0 +1,16 @@+import pytest+from yarl import URL++from crawl4ai.async_configs import ProxyConfig+from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy++[email protected]("password", ["plainpass", "p@ss", "p#ss", "pa/ss", "p?ss", "p%41ss"])+def test_proxy_url_roundtrips_credentials(password):+ strategy = AsyncHTTPCrawlerStrategy()+ config = ProxyConfig(server="http://host:8080", username="us:er", password=password)+ url = URL(strategy._format_proxy_url(config))+ assert url.host == "host"+ assert url.port == 8080+ assert url.user == "us:er"+ assert url.password == password