Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(setup): give the real fix when the browser install fails, not a cloud link
unclecode129a950ca6cc
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5bcb58aa..83701305 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Changed++- `crawl4ai-setup` no longer points to Crawl4AI Cloud when the browser install fails. It retries the browser without `--with-deps` (which needs root on Linux), prints the exact command for the system libraries, and exits with code 1 instead of printing "completed". The cloud banner at the end shows only after a successful setup.+- `crawl4ai-doctor` exits with code 1 when its test crawl fails, counts a blocked page as a failure, and shows no cloud banner.+- The cloud banner and the blocked line state the current free offer: 1,000 free credits every month.+### Crawl4AI Cloud- Requests with `proxy: isp` and a country (for example UK pricing pages) no longer fail with "no-proxy-exit" after the address set is renewed. Thanks to [Summone](https://summone.co.uk) for the report.diff --git a/crawl4ai/cloud_notice.py b/crawl4ai/cloud_notice.pyindex b5814578..e0ace093 100644--- a/crawl4ai/cloud_notice.py+++ b/crawl4ai/cloud_notice.py@@ -1,13 +1,16 @@"""The Crawl4AI Cloud notice: a short yellow banner that tells a user the hosted version exists.-It prints in four places:+It prints in three places:-- at the end of ``crawl4ai-setup``, every time (``show_at_install``);+- at the end of a successful ``crawl4ai-setup``, every time (``show_at_install``);- when the first ``AsyncWebCrawler`` starts or the ``crwl`` CLI runs (``show_once``): once a day permachine while the launch offer runs (before ``DAILY_UNTIL``), then once per version per machine, apatch release included. A marker file in the crawl4ai home folder holds the day, or the version, last shown;-- when a crawl is blocked by anti-bot protection, at most once a day (``show_blocked``);-- when ``crawl4ai-setup`` could not install the browser (``show_setup_failed``).+- when a crawl is blocked by anti-bot protection, at most once a day (``show_blocked``).++A failure of the library itself (a setup that did not install the browser, a crash, ``crawl4ai-doctor``)+never points to the cloud: it gets the real error and the real fix. The cloud is named only where the+site, not the library, is the limit.The banner goes to the logger's console (stderr by default), never to stdout, so a piped ``crwl`` outputor an MCP stdio transport stays clean. Rich drops the color when the output is not a terminal and honours@@ -24,7 +27,6 @@ from pathlib import PathCLOUD_URL = "https://crawl4ai.com/?ref=pip"BLOCKED_URL = "https://crawl4ai.com/?ref=blocked"-SETUP_URL = "https://crawl4ai.com/?ref=setup"_ENV_OFF = "CRAWL4AI_NO_CLOUD_NOTICE"_MARKER = ".cloud-notice-shown"_BLOCKED_MARKER = ".cloud-notice-blocked"@@ -35,21 +37,16 @@ _RULE = "=" * 72LINES = ("Crawl4AI Cloud is live: the same crawler, hosted.",- "Free credit to start, no card. Soft launch: prices can change, what you buy stays yours.",+ "1,000 free credits every month, no card.",f"Get a key in 10 seconds: {CLOUD_URL}",f"The library stays open source, forever. Hide this banner: {_ENV_OFF}=1",)BLOCKED_LINES = (- "Blocked? Crawl4AI Cloud gets through the hard sites, free credit to start.",+ "Blocked? Crawl4AI Cloud gets through the hard sites, 1,000 free credits every month.",f"{BLOCKED_URL} Hide this banner: {_ENV_OFF}=1",)-SETUP_LINES = (- "Browser setup failed? Skip it: the same crawler, hosted, nothing to install, free credit to start.",- SETUP_URL,-)-def _home() -> Path:try:@@ -114,7 +111,7 @@ def _claim(name: str, value: str) -> bool:def show_at_install(logger=None) -> None:- """Print the banner at the end of ``crawl4ai-setup``. Every run; nothing is remembered."""+ """Print the banner at the end of a successful ``crawl4ai-setup``. Every run; nothing is remembered."""if is_off():return_print(LINES, logger)@@ -140,10 +137,3 @@ def show_blocked(logger=None) -> bool:return False_print(BLOCKED_LINES, logger)return True---def show_setup_failed(logger=None) -> None:- """Print the setup banner when ``crawl4ai-setup`` could not install the browser."""- if is_off():- return- _print(SETUP_LINES, logger)diff --git a/crawl4ai/install.py b/crawl4ai/install.pyindex 515201db..3d203264 100644--- a/crawl4ai/install.py+++ b/crawl4ai/install.py@@ -39,27 +39,19 @@ def setup_home_directory():with open(crawl4ai_config, "w") as f:f.write("")-def _cloud_setup_hint():- """After a failed browser install, name the hosted crawler that needs no browser."""- try:- from .cloud_notice import show_setup_failed-- show_setup_failed(logger)- except Exception:- pass--def post_install():"""Run all post-installation tasks.Checks CRAWL4AI_MODE environment variable. If set to 'api',skips Playwright browser installation.+ Exits with code 1 when the browser could not be installed."""logger.info("Running post-installation setup...", tag="INIT")setup_home_directory()# Check environment variable to conditionally skip Playwright installrun_mode = os.getenv('CRAWL4AI_MODE')+ browser_installed = Trueif run_mode == 'api':logger.warning("CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.",@@ -67,11 +59,18 @@ def post_install():)else:# Proceed with installation only if mode is not 'api'- install_playwright()+ browser_installed = install_playwright()run_migration()# TODO: Will be added in the future# setup_builtin_browser()+ if not browser_installed:+ logger.error(+ "Setup did not finish: the browser is not installed. "+ "Run the command above, then 'crawl4ai-doctor' to check.",+ tag="SETUP",+ )+ sys.exit(1)logger.success("Post-installation setup completed!", tag="COMPLETE")try:from .cloud_notice import show_at_install@@ -107,62 +106,65 @@ async def _setup_builtin_browser():raise-def install_playwright():- logger.info("Installing Playwright browsers...", tag="INIT")+def _run_install(command: list) -> str:+ """Run one install command. Returns "" when it worked, otherwise the reason it failed."""try:- # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])- subprocess.check_call(- [- sys.executable,- "-m",- "playwright",- "install",- "--with-deps",- "--force",- "chromium",- ]- )- logger.success(- "Playwright installation completed successfully.", tag="COMPLETE"- )- except subprocess.CalledProcessError:- # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")- logger.warning(- f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."- )- _cloud_setup_hint()- except Exception:- # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")- logger.warning(- f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."- )- _cloud_setup_hint()-+ subprocess.check_call(command)+ return ""+ except subprocess.CalledProcessError as e:+ return f"exit code {e.returncode}"+ except Exception as e:+ return str(e)+++def _install_chromium(tool: str) -> bool:+ """Install Chromium with ``tool`` ("playwright" or "patchright"). Returns True when the browser is installed.++ 1. Try with ``--with-deps``, which also installs the system libraries. On Linux that needs root.+ 2. If that fails, install the browser alone, and print the command for the system libraries.+ 3. If that fails too, print the command to run by hand.+ """+ install = [sys.executable, "-m", tool, "install"]+ reason = _run_install(install + ["--with-deps", "--force", "chromium"])+ if not reason:+ logger.success(f"{tool} Chromium installed.", tag="COMPLETE")+ return True+ logger.warning(+ f"'{tool} install --with-deps chromium' failed ({reason}); the error is printed above. "+ "Trying the browser alone, without the system libraries...",+ tag="SETUP",+ )++ # ---- 2. The browser alone needs no root; only the system libraries do+ reason = _run_install(install + ["--force", "chromium"])+ if not reason:+ logger.warning(f"{tool} Chromium installed, but its system libraries may be missing.", tag="SETUP")+ if sys.platform.startswith("linux"):+ logger.warning(+ f"If the browser does not start, run: sudo {sys.executable} -m {tool} install-deps chromium",+ tag="SETUP",+ )+ return True++ # ---- 3. Nothing worked: the user runs it by hand+ logger.error(f"'{tool} install chromium' failed ({reason}); the error is printed above.", tag="SETUP")+ logger.error(f"Install it by hand: {sys.executable} -m {tool} install --with-deps chromium", tag="SETUP")+ if sys.platform.startswith("linux"):+ logger.error("Installing the system libraries needs root: run the command with sudo.", tag="SETUP")+ return False+++def install_playwright() -> bool:+ """Install the Playwright and Patchright browsers. Returns True when the Playwright browser is installed.+ Patchright only serves the undetected mode, so its failure is a warning."""+ logger.info("Installing Playwright browsers...", tag="INIT")+ playwright_installed = _install_chromium("playwright")+# Install Patchright browsers for undetected browser supportlogger.info("Installing Patchright browsers for undetected mode...", tag="INIT")- try:- subprocess.check_call(- [- sys.executable,- "-m",- "patchright",- "install",- "--with-deps",- "--force",- "chromium",- ]- )- logger.success(- "Patchright installation completed successfully.", tag="COMPLETE"- )- except subprocess.CalledProcessError:- logger.warning(- f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."- )- except Exception:- logger.warning(- f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."- )+ if not _install_chromium("patchright"):+ logger.warning("Only the undetected browser mode needs Patchright; everything else works without it.", tag="SETUP")+ return playwright_installeddef run_migration():@@ -211,11 +213,11 @@ async def run_doctor():logger.info("Testing crawling capabilities...", tag="TEST")result = await crawler.arun(url="https://crawl4ai.com", config=run_config)- if result and result.markdown:+ if result and result.success and result.markdown:logger.success("✅ Crawling test passed!", tag="COMPLETE")return Trueelse:- raise Exception("Failed to get content")+ raise Exception(f"Failed to get content: {getattr(result, 'error_message', None) or 'empty page'}")except Exception as e:logger.error(f"❌ Test failed: {e}", tag="ERROR")@@ -223,8 +225,12 @@ async def run_doctor():def doctor():- """Entry point for the doctor command"""+ """Entry point for the doctor command. Exits with code 1 when the test crawl fails."""import asyncio- asyncio.run(run_doctor())- sys.exit(0)+ # The doctor checks the installation; a cloud banner here would read as "the library is broken, go hosted"+ from .cloud_notice import _ENV_OFF++ os.environ[_ENV_OFF] = "1"+ passed = asyncio.run(run_doctor())+ sys.exit(0 if passed else 1)diff --git a/docs/md_v2/advanced/anti-bot-and-fallback.md b/docs/md_v2/advanced/anti-bot-and-fallback.mdindex 68ed03de..7b90ab55 100644--- a/docs/md_v2/advanced/anti-bot-and-fallback.md+++ b/docs/md_v2/advanced/anti-bot-and-fallback.md@@ -2,6 +2,15 @@When crawling sites protected by anti-bot systems (Akamai, Cloudflare, PerimeterX, DataDome, Imperva, etc.), requests often get blocked with CAPTCHAs, 403 responses, or empty pages. Crawl4AI provides a layered retry and fallback system that automatically detects blocking and escalates through multiple strategies until content is retrieved.+!!! tip "Hard sites? One Cloud call"+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-antibot) runs the detection, retries and fallbacks for you, on its own browsers and addresses: send a URL, get clean Markdown. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-antibot)++ ```bash+ curl -X POST https://api.crawl4ai.com/scrape \+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \+ -d '{"url": "https://example.com", "format": "md"}'+ ```+## How Detection WorksAfter each crawl attempt, Crawl4AI inspects the HTTP status code and HTML content for known anti-bot signals:diff --git a/docs/md_v2/advanced/proxy-security.md b/docs/md_v2/advanced/proxy-security.mdindex 39c5d7ad..5105dda7 100644--- a/docs/md_v2/advanced/proxy-security.md+++ b/docs/md_v2/advanced/proxy-security.md@@ -2,6 +2,15 @@This guide covers proxy configuration and security features in Crawl4AI, including SSL certificate analysis and proxy rotation strategies.+!!! tip "No proxies to buy or rotate"+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-proxy) fetches from the country you choose with its own addresses: add `"country": "gb"` to a call, nothing to configure. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-proxy)++ ```bash+ curl -X POST https://api.crawl4ai.com/scrape \+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \+ -d '{"url": "https://example.com", "format": "md"}'+ ```+## Understanding Proxy ConfigurationCrawl4AI recommends configuring proxies per request through `CrawlerRunConfig.proxy_config`. This gives you precise control, enables rotation strategies, and keeps examples simple enough to copy, paste, and run.diff --git a/docs/md_v2/advanced/undetected-browser.md b/docs/md_v2/advanced/undetected-browser.mdindex c709602f..6d178762 100644--- a/docs/md_v2/advanced/undetected-browser.md+++ b/docs/md_v2/advanced/undetected-browser.md@@ -1,5 +1,14 @@# Undetected Browser Mode+!!! tip "No browser to patch"+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-stealth) picks the right way to fetch each page, a real browser included, and gets past the usual bot checks for you. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-stealth)++ ```bash+ curl -X POST https://api.crawl4ai.com/scrape \+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \+ -d '{"url": "https://example.com", "format": "md"}'+ ```+## OverviewCrawl4AI offers two powerful anti-bot features to help you access websites with bot detection:diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.mdindex 5f6ae1c4..54ce0a19 100644--- a/docs/md_v2/core/self-hosting.md+++ b/docs/md_v2/core/self-hosting.md@@ -13,6 +13,15 @@> answer with *connection reset* even though the container reports healthy.> Every quickstart below therefore starts by setting a token.+!!! tip "Don't want to run the server?"+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-selfhost) is the same engine, hosted: no Docker, no browsers, one key over HTTPS. Self-hosting stays fully supported. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-selfhost)++ ```bash+ curl -X POST https://api.crawl4ai.com/scrape \+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \+ -d '{"url": "https://example.com", "format": "md"}'+ ```+**Take Control of Your Web Crawling Infrastructure**Self-hosting Crawl4AI gives you complete control over your web crawling and data extraction pipeline. Unlike cloud-based solutions, you own your data, infrastructure, and destiny.diff --git a/docs/md_v2/overrides/partials/cloud-banner.html b/docs/md_v2/overrides/partials/cloud-banner.htmlindex 339e295e..9871c33d 100644--- a/docs/md_v2/overrides/partials/cloud-banner.html+++ b/docs/md_v2/overrides/partials/cloud-banner.html@@ -10,13 +10,13 @@<div class="c4b-text"><div class="c4b-eyebrow"><span class="c4b-tag">New</span> good news for everyone who runs Crawl4AI</div><h2 class="c4b-title">Crawl4AI <em>Cloud</em> is live.<br>The same crawler, hosted.</h2>- <p class="c4b-lead">No browsers to run, no proxies to buy, no cookies to babysit. One key with <b>free credit to start</b>, and your AI agent can <b>scrape, extract, search, read Google's results page (SERP) and answer questions</b> through the same engine you already use. Free: 10 a minute, 2 at a time, 2,000 credits to start. Paid: 120 a minute, 10 at a time, from $10.</p>+ <p class="c4b-lead">No browsers to run, no proxies to buy, no cookies to babysit. One key with <b>1,000 free credits every month</b>, and your AI agent can <b>scrape, extract, search, read Google's results page (SERP) and answer questions</b> through the same engine you already use. Free: 1,000 credits every month, 10 a minute, 2 at a time. Paid: 120 a minute, 10 at a time, from $10 a month.</p><p class="c4b-oss"><b>Always open source, forever.</b> The library stays here and stays free. The cloud is for the days you want to build fast and let us run the infrastructure.</p><div class="c4b-cta">- <a class="c4b-btn" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_key_clicked">Get a key, free credit to start →</a>+ <a class="c4b-btn" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_key_clicked">Get a key, 1,000 free credits a month →</a><a class="c4b-btn c4b-ghost" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_cloud_clicked">See Crawl4AI Cloud</a></div>- <div class="c4b-meta"><span>no card</span><span><b>free credit</b> to start · soft launch: prices can change, what you buy stays yours</span><span>EU · Singapore · US</span><span>MCP built in</span></div>+ <div class="c4b-meta"><span>no card</span><span><b>1,000 free credits</b> every month</span><span>EU · Singapore · US</span><span>MCP built in</span></div></div><div class="c4b-paste" aria-label="Paste this into your agent"><div class="c4b-tabs" role="tablist">@@ -36,7 +36,7 @@<div class="c4b-small"><span class="c4b-star">★</span>- <span class="c4b-t"><b>Crawl4AI Cloud is live</b> — the same crawler, hosted. Get a key with free credit to start and paste one line into your agent.</span>+ <span class="c4b-t"><b>Crawl4AI Cloud is live</b> — the same crawler, hosted. 1,000 free credits every month; paste one line into your agent.</span><span class="c4b-sp"></span><a class="c4b-go" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_key_clicked">Get a key →</a><button type="button" data-c4b-act="expand">show</button>diff --git a/tests/unit/test_cloud_notice.py b/tests/unit/test_cloud_notice.pyindex 555f30f9..0749bf2a 100644--- a/tests/unit/test_cloud_notice.py+++ b/tests/unit/test_cloud_notice.py@@ -61,7 +61,7 @@ def test_the_blocked_banner_prints_once_a_day(home, monkeypatch):def test_the_banner_is_ascii_with_the_link_and_the_off_switch():- for lines in (cloud_notice.LINES, cloud_notice.BLOCKED_LINES, cloud_notice.SETUP_LINES):+ for lines in (cloud_notice.LINES, cloud_notice.BLOCKED_LINES):shown = _render(lines)assert shown.isascii(), "every console can print it"assert shown.startswith(cloud_notice._RULE) and shown.rstrip().endswith(cloud_notice._RULE)diff --git a/tests/unit/test_install_setup.py b/tests/unit/test_install_setup.pynew file mode 100644index 00000000..ea835b99--- /dev/null+++ b/tests/unit/test_install_setup.py@@ -0,0 +1,82 @@+"""crawl4ai-setup and crawl4ai-doctor: a failure gets the real fix and a real exit code, never the cloud."""++import subprocess++import pytest++from crawl4ai import cloud_notice, install+++class _Installs:+ """Stands in for subprocess.check_call: records each command and fails the ones asked to fail."""++ def __init__(self, fail_with_deps=False, fail_alone=False):+ self.commands = []+ self.fail_with_deps = fail_with_deps+ self.fail_alone = fail_alone++ def __call__(self, command):+ self.commands.append(command)+ with_deps = "--with-deps" in command+ if (with_deps and self.fail_with_deps) or (not with_deps and self.fail_alone):+ raise subprocess.CalledProcessError(1, command)+ return 0+++def test_the_browser_installs_with_its_system_libraries(monkeypatch):+ installs = _Installs()+ monkeypatch.setattr(install.subprocess, "check_call", installs)+ assert install._install_chromium("playwright") is True+ assert len(installs.commands) == 1 and "--with-deps" in installs.commands[0]+++def test_without_root_the_browser_installs_alone(monkeypatch):+ installs = _Installs(fail_with_deps=True)+ monkeypatch.setattr(install.subprocess, "check_call", installs)+ assert install._install_chromium("playwright") is True+ assert "--with-deps" not in installs.commands[1], "the second try needs no root"+++def test_nothing_installs_returns_false(monkeypatch):+ monkeypatch.setattr(install.subprocess, "check_call", _Installs(fail_with_deps=True, fail_alone=True))+ assert install._install_chromium("playwright") is False+++def test_a_failed_setup_exits_1_and_shows_no_banner(monkeypatch):+ banners = []+ monkeypatch.delenv("CRAWL4AI_MODE", raising=False)+ monkeypatch.setattr(install, "setup_home_directory", lambda: None)+ monkeypatch.setattr(install, "run_migration", lambda: None)+ monkeypatch.setattr(install, "install_playwright", lambda: False)+ monkeypatch.setattr(cloud_notice, "_print", lambda lines, logger=None: banners.append(lines))+ with pytest.raises(SystemExit) as exit_info:+ install.post_install()+ assert exit_info.value.code == 1+ assert banners == []+++def test_a_good_setup_shows_the_banner(monkeypatch):+ banners = []+ monkeypatch.delenv("CRAWL4AI_MODE", raising=False)+ monkeypatch.delenv(cloud_notice._ENV_OFF, raising=False)+ monkeypatch.setattr(install, "setup_home_directory", lambda: None)+ monkeypatch.setattr(install, "run_migration", lambda: None)+ monkeypatch.setattr(install, "install_playwright", lambda: True)+ monkeypatch.setattr(cloud_notice, "_print", lambda lines, logger=None: banners.append(lines))+ install.post_install()+ assert banners == [cloud_notice.LINES]++[email protected]("passed, code", [(True, 0), (False, 1)])+def test_the_doctor_exit_code_follows_the_test_and_hides_the_cloud(monkeypatch, passed, code):+ # setenv first, so pytest restores the variable after doctor() writes it+ monkeypatch.setenv(cloud_notice._ENV_OFF, "0")++ async def fake_doctor():+ return passed++ monkeypatch.setattr(install, "run_doctor", fake_doctor)+ with pytest.raises(SystemExit) as exit_info:+ install.doctor()+ assert exit_info.value.code == code+ assert cloud_notice.is_off()