Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): stop launching Chromium with certificate checks off
ntohidif0ada4c7cd2b
diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.pyindex 17d6f5eb..fa671c69 100644--- a/crawl4ai/browser_manager.py+++ b/crawl4ai/browser_manager.py@@ -41,6 +41,20 @@ BROWSER_DISABLE_OPTIONS = []+def chromium_sandbox_requested() -> bool:+ """True when the operator opts into the Chromium sandbox+ (CRAWL4AI_CHROMIUM_SANDBOX=true). The default keeps --no-sandbox because+ Chromium cannot start sandboxed in a container without user namespaces."""+ return os.environ.get("CRAWL4AI_CHROMIUM_SANDBOX", "false").lower() == "true"+++def apply_sandbox_policy(args: List[str]) -> List[str]:+ """Drop --no-sandbox, also from extra_args, when the sandbox is requested."""+ if chromium_sandbox_requested():+ return [a for a in args if a != "--no-sandbox"]+ return args++class ManagedBrowser:"""Manages the browser process and context. This class allows to connect to the browser using CDP protocol.@@ -76,8 +90,6 @@ class ManagedBrowser:"--no-default-browser-check","--disable-infobars","--window-position=0,0",- "--ignore-certificate-errors",- "--ignore-certificate-errors-spki-list","--disable-blink-features=AutomationControlled","--window-position=400,0","--disable-renderer-backgrounding",@@ -124,7 +136,7 @@ class ManagedBrowser:elif config.proxy_config:flags.append(f"--proxy-server={config.proxy_config.server}")# dedupe- return list(dict.fromkeys(flags))+ return apply_sandbox_policy(list(dict.fromkeys(flags)))browser_type: struser_data_dir: str@@ -197,6 +209,7 @@ class ManagedBrowser:if self.browser_config.extra_args:args.extend(self.browser_config.extra_args)+ args = apply_sandbox_policy(args)# ── make sure no old Chromium instance is owning the same port/profile ──@@ -395,6 +408,10 @@ class ManagedBrowser:flags.append(f"--window-size={self.browser_config.viewport_width},{self.browser_config.viewport_height}")# merge common launch flagsflags.extend(self.build_browser_flags(self.browser_config))+ # The default context of a CDP-connected browser takes no+ # ignore_https_errors option, so insecure mode needs the launch flag.+ if self.browser_config.ignore_https_errors:+ flags.append("--ignore-certificate-errors")elif self.browser_type == "firefox":flags = ["--remote-debugging-port",@@ -867,7 +884,7 @@ class BrowserManager:launch_kwargs = {"headless": self.config.headless,- "args": list(dict.fromkeys(cli_args)), # dedupe+ "args": apply_sandbox_policy(list(dict.fromkeys(cli_args))), # dedupe"viewport": {"width": self.config.viewport_width,"height": self.config.viewport_height,@@ -876,6 +893,8 @@ class BrowserManager:"ignore_https_errors": self.config.ignore_https_errors,"accept_downloads": self.config.accept_downloads,}+ if chromium_sandbox_requested():+ launch_kwargs["chromium_sandbox"] = Trueif self.config.proxy_config:launch_kwargs["proxy"] = {@@ -1095,8 +1114,6 @@ class BrowserManager:"--no-default-browser-check","--disable-infobars","--window-position=0,0",- "--ignore-certificate-errors",- "--ignore-certificate-errors-spki-list","--disable-blink-features=AutomationControlled","--window-position=400,0","--disable-renderer-backgrounding",@@ -1139,9 +1156,12 @@ class BrowserManager:args.extend(self.config.extra_args)# Deduplicate args- args = list(dict.fromkeys(args))+ args = apply_sandbox_policy(list(dict.fromkeys(args)))browser_args = {"headless": self.config.headless, "args": args}+ # Playwright adds --no-sandbox itself unless chromium_sandbox is True.+ if chromium_sandbox_requested():+ browser_args["chromium_sandbox"] = True# On Windows, passing channel='chromium' (the default) causes Playwright# to look for a system Chrome installation instead of using the bundleddiff --git a/deploy/docker/server.py b/deploy/docker/server.pyindex 24815527..f42460ce 100644--- a/deploy/docker/server.py+++ b/deploy/docker/server.py@@ -102,6 +102,8 @@ EXECUTE_JS_ENABLED = os.environ.get("CRAWL4AI_EXECUTE_JS_ENABLED", "false").lowe# an unprivileged user namespace (unprivileged_userns_clone=1) or a seccomp# profile, set CRAWL4AI_CHROMIUM_SANDBOX=true to drop --no-sandbox and run the# renderer sandboxed. Verify Chromium still starts after flipping it.+# crawl4ai.browser_manager reads the same variable at launch, so the sandbox+# also applies to browsers built from a request body, not only this config.CHROMIUM_SANDBOX = os.environ.get("CRAWL4AI_CHROMIUM_SANDBOX", "false").lower() == "true"# Warn loudly if API token is not set (all endpoints unauthenticated)diff --git a/deploy/docker/tests/test_security_default_posture.py b/deploy/docker/tests/test_security_default_posture.pyindex 96c50afa..1f4b1482 100644--- a/deploy/docker/tests/test_security_default_posture.py+++ b/deploy/docker/tests/test_security_default_posture.py@@ -162,10 +162,23 @@ class TestDefaultDeployIsSafe:"default config launches Chromium with --no-sandbox")- def test_tls_verification_not_disabled(self, effective_browser_args):- """TLS verification must not be disabled by default."""- assert "--ignore-certificate-errors" not in effective_browser_args- assert "--allow-insecure-localhost" not in effective_browser_args+ def test_tls_verification_not_disabled(self, effective_browser_args, server_module):+ """TLS verification must not be disabled by default.++ Checks the flags Chromium is really launched with, not only+ config.yml: the library adds its own launch flags on every path."""+ from crawl4ai import BrowserConfig+ from crawl4ai.browser_manager import BrowserManager, ManagedBrowser++ for bc in (server_module.get_default_browser_config(), BrowserConfig()):+ launched = (+ list(effective_browser_args)+ + BrowserManager(browser_config=bc)._build_browser_args()["args"]+ + ManagedBrowser.build_browser_flags(bc)+ )+ assert "--ignore-certificate-errors" not in launched+ assert "--ignore-certificate-errors-spki-list" not in launched+ assert "--allow-insecure-localhost" not in launched# ───────────────────────── R7: deploy posture ─────────────────────────diff --git a/deploy/docker/utils.py b/deploy/docker/utils.pyindex 0fc2e69c..b1cb709b 100644--- a/deploy/docker/utils.py+++ b/deploy/docker/utils.py@@ -72,8 +72,6 @@ DEFAULT_CONFIG = {"--disable-dev-shm-usage","--disable-gpu","--disable-software-rasterizer",- "--allow-insecure-localhost",- "--ignore-certificate-errors",],},},diff --git a/tests/unit/test_browser_tls_sandbox_flags.py b/tests/unit/test_browser_tls_sandbox_flags.pynew file mode 100644index 00000000..6034ffda--- /dev/null+++ b/tests/unit/test_browser_tls_sandbox_flags.py@@ -0,0 +1,139 @@+"""TLS verification and Chromium sandbox launch flags.++ignore_https_errors=False must really verify certificates. A launch flag like+--ignore-certificate-errors turns verification off for the whole browser, so+the context option would have no effect. The end-to-end tests crawl a local+HTTPS server with a self-signed certificate on every launch path.++CRAWL4AI_CHROMIUM_SANDBOX=true must run Chromium with its sandbox on.+"""++import asyncio+import http.server+import secrets+import ssl+import subprocess+import threading++import pytest++from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig+from crawl4ai.browser_manager import BrowserManager, ManagedBrowser++CERT_FLAGS = ("--ignore-certificate-errors", "--ignore-certificate-errors-spki-list")+++def _has_cert_flag(args):+ return any(a.split("=", 1)[0] in CERT_FLAGS for a in args)+++# ── launch flag construction (no browser) ──────────────────────────────++[email protected]("ignore", [True, False])+def test_build_browser_args_has_no_cert_flag(ignore):+ bc = BrowserConfig(headless=True, ignore_https_errors=ignore)+ assert not _has_cert_flag(BrowserManager(browser_config=bc)._build_browser_args()["args"])++[email protected]("ignore", [True, False])+def test_build_browser_flags_has_no_cert_flag(ignore):+ bc = BrowserConfig(headless=True, ignore_https_errors=ignore)+ assert not _has_cert_flag(ManagedBrowser.build_browser_flags(bc))++[email protected]("ignore", [True, False])+def test_managed_subprocess_cert_flag_follows_config(ignore, monkeypatch):+ """The CDP default context has no ignore_https_errors option, so the+ subprocess launch carries the flag only when the config asks for it."""+ bc = BrowserConfig(headless=True, ignore_https_errors=ignore)+ mb = ManagedBrowser(browser_type="chromium", headless=True, browser_config=bc)++ async def fake_path():+ return "chromium"++ monkeypatch.setattr(mb, "_get_browser_path", fake_path)+ args = asyncio.run(mb._get_browser_args())+ assert _has_cert_flag(args) is ignore+++def test_sandbox_off_by_default(monkeypatch):+ monkeypatch.delenv("CRAWL4AI_CHROMIUM_SANDBOX", raising=False)+ bc = BrowserConfig(headless=True)+ launch = BrowserManager(browser_config=bc)._build_browser_args()+ assert "--no-sandbox" in launch["args"]+ assert not launch.get("chromium_sandbox")+ assert "--no-sandbox" in ManagedBrowser.build_browser_flags(bc)+++def test_sandbox_requested_by_env(monkeypatch):+ monkeypatch.setenv("CRAWL4AI_CHROMIUM_SANDBOX", "true")+ bc = BrowserConfig(headless=True, extra_args=["--no-sandbox", "--disable-gpu"])+ launch = BrowserManager(browser_config=bc)._build_browser_args()+ assert "--no-sandbox" not in launch["args"]+ assert "--disable-gpu" in launch["args"]+ assert launch["chromium_sandbox"] is True+ assert "--no-sandbox" not in ManagedBrowser.build_browser_flags(bc)+++# ── end to end: self-signed HTTPS origin ───────────────────────────────++[email protected](scope="module")+def self_signed_origin(tmp_path_factory):+ work = tmp_path_factory.mktemp("tls")+ subprocess.run(+ ["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1",+ "-subj", "/CN=localhost", "-keyout", str(work / "k.pem"), "-out", str(work / "c.pem")],+ check=True, capture_output=True,+ )+ marker = "tls-marker-" + secrets.token_hex(6)+ # Padding keeps the page above the anti-bot near-empty threshold.+ filler = "<p>" + "Plain test content. " * 100 + "</p>"+ (work / "index.html").write_text(+ f"<html><head><title>tls</title></head><body><p>{marker}</p>{filler}</body></html>"+ )++ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)+ ctx.load_cert_chain(str(work / "c.pem"), str(work / "k.pem"))++ class Handler(http.server.SimpleHTTPRequestHandler):+ def __init__(self, *a, **k):+ super().__init__(*a, directory=str(work), **k)++ def log_message(self, *a):+ pass++ srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ srv.socket = ctx.wrap_socket(srv.socket, server_side=True)+ threading.Thread(target=srv.serve_forever, daemon=True).start()+ yield f"https://localhost:{srv.server_address[1]}/", marker+ srv.shutdown()+++async def _crawl(url, **browser_kwargs):+ bc = BrowserConfig(headless=True, verbose=False, **browser_kwargs)+ async with AsyncWebCrawler(config=bc) as crawler:+ r = await crawler.arun(url, config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))+ return r.html or ""+++LAUNCH_MODES = {+ "playwright_launch": {},+ "managed_cdp": {"use_managed_browser": True},+ "persistent_context": {"use_persistent_context": True},+}++[email protected]("mode", list(LAUNCH_MODES))+def test_tls_verified_when_ignore_https_errors_false(self_signed_origin, mode):+ url, marker = self_signed_origin+ html = asyncio.run(_crawl(url, ignore_https_errors=False, **LAUNCH_MODES[mode]))+ assert marker not in html, "untrusted certificate accepted with ignore_https_errors=False"++[email protected]("mode", list(LAUNCH_MODES))+def test_insecure_tls_when_ignore_https_errors_true(self_signed_origin, mode):+ url, marker = self_signed_origin+ html = asyncio.run(_crawl(url, ignore_https_errors=True, **LAUNCH_MODES[mode]))+ assert marker in html, "ignore_https_errors=True no longer reaches a self-signed origin"