Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Feature/self hosted proxy support (#2115)
Piyushcebb04c736b4
diff --git a/skyvern/config.py b/skyvern/config.pyindex 992cf5c6b..5fca961c2 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -44,6 +44,10 @@ class Settings(BaseSettings):BLOCKED_HOSTS: list[str] = ["localhost"]ALLOWED_HOSTS: list[str] = []+ # Format: "http://<username>:<password>@host:port, http://<username>:<password>@host:port, ...."+ HOSTED_PROXY_POOL: str = ""+ ENABLE_PROXY: bool = False+# Secret key for JWT. Please generate your own secret key in productionSECRET_KEY: str = "PLACEHOLDER"# Algorithm used to sign the JWTdiff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex 0131dbd9e..5faf84a75 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -2,6 +2,8 @@ from __future__ import annotationsimport asyncioimport os+import random+import reimport socketimport subprocessimport time@@ -9,6 +11,7 @@ import uuidfrom datetime import datetimefrom pathlib import Pathfrom typing import Any, Awaitable, Callable, Protocol+from urllib.parse import urlparseimport aiofilesimport psutil@@ -210,6 +213,11 @@ class BrowserContextFactory:},}+ if settings.ENABLE_PROXY:+ proxy_config = setup_proxy()+ if proxy_config:+ args["proxy"] = proxy_config+if proxy_location:if tz_info := get_tzinfo_from_proxy(proxy_location=proxy_location):args["timezone_id"] = tz_info.key@@ -311,6 +319,63 @@ class BrowserArtifacts(BaseModel):return await f.read()+def setup_proxy() -> dict | None:+ if not settings.HOSTED_PROXY_POOL or settings.HOSTED_PROXY_POOL.strip() == "":+ LOG.warning("No proxy server value found. Continuing without using proxy...")+ return None++ proxy_servers = [server.strip() for server in settings.HOSTED_PROXY_POOL.split(",") if server.strip()]++ if not proxy_servers:+ LOG.warning("Proxy pool contains only empty values. Continuing without proxy...")+ return None++ valid_proxies = []+ for proxy in proxy_servers:+ if _is_valid_proxy_url(proxy):+ valid_proxies.append(proxy)+ else:+ LOG.warning(f"Invalid proxy URL format: {proxy}")++ if not valid_proxies:+ LOG.warning("No valid proxy URLs found. Continuing without proxy...")+ return None++ try:+ proxy_server = random.choice(valid_proxies)+ proxy_creds = _get_proxy_server_creds(proxy_server)++ LOG.info("Found proxy server creds, using them...")++ return {+ "server": proxy_server,+ "username": proxy_creds.get("username", ""),+ "password": proxy_creds.get("password", ""),+ }+ except Exception as e:+ LOG.warning(f"Error setting up proxy: {e}. Continuing without proxy...")+ return None+++def _is_valid_proxy_url(url: str) -> bool:+ PROXY_PATTERN = re.compile(r"^(http|https|socks5):\/\/([^:@]+(:[^@]*)?@)?[^\s:\/]+(:\d+)?$")+ try:+ parsed = urlparse(url)+ if not parsed.scheme or not parsed.netloc:+ return False+ return bool(PROXY_PATTERN.match(url))+ except Exception:+ return False+++def _get_proxy_server_creds(proxy: str) -> dict:+ parsed_url = urlparse(proxy)+ if parsed_url.username and parsed_url.password:+ return {"username": parsed_url.username, "password": parsed_url.password}+ LOG.warning("No credentials found in the proxy URL.")+ return {}++def _get_cdp_port(kwargs: dict) -> int | None:raw_cdp_port = kwargs.get("cdp_port")if isinstance(raw_cdp_port, (int, str)):