Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(setup): give the real fix when the browser install fails, not a cloud link

unclecode129a950ca6cc
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5bcb58aa..83701305 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Changed
+
+- `crawl4ai-setup` no longer points to Crawl4AI Cloud when the browser install fails. It retries the browser without `--with-deps` (which needs root on Linux), prints the exact command for the system libraries, and exits with code 1 instead of printing "completed". The cloud banner at the end shows only after a successful setup.
+- `crawl4ai-doctor` exits with code 1 when its test crawl fails, counts a blocked page as a failure, and shows no cloud banner.
+- The cloud banner and the blocked line state the current free offer: 1,000 free credits every month.
+
### Crawl4AI Cloud
- Requests with `proxy: isp` and a country (for example UK pricing pages) no longer fail with "no-proxy-exit" after the address set is renewed. Thanks to [Summone](https://summone.co.uk) for the report.
diff --git a/crawl4ai/cloud_notice.py b/crawl4ai/cloud_notice.py
index b5814578..e0ace093 100644
--- a/crawl4ai/cloud_notice.py
+++ b/crawl4ai/cloud_notice.py
@@ -1,13 +1,16 @@
"""The Crawl4AI Cloud notice: a short yellow banner that tells a user the hosted version exists.
-It prints in four places:
+It prints in three places:
-- at the end of ``crawl4ai-setup``, every time (``show_at_install``);
+- at the end of a successful ``crawl4ai-setup``, every time (``show_at_install``);
- when the first ``AsyncWebCrawler`` starts or the ``crwl`` CLI runs (``show_once``): once a day per
machine while the launch offer runs (before ``DAILY_UNTIL``), then once per version per machine, a
patch release included. A marker file in the crawl4ai home folder holds the day, or the version, last shown;
-- when a crawl is blocked by anti-bot protection, at most once a day (``show_blocked``);
-- when ``crawl4ai-setup`` could not install the browser (``show_setup_failed``).
+- when a crawl is blocked by anti-bot protection, at most once a day (``show_blocked``).
+
+A failure of the library itself (a setup that did not install the browser, a crash, ``crawl4ai-doctor``)
+never points to the cloud: it gets the real error and the real fix. The cloud is named only where the
+site, not the library, is the limit.
The banner goes to the logger's console (stderr by default), never to stdout, so a piped ``crwl`` output
or an MCP stdio transport stays clean. Rich drops the color when the output is not a terminal and honours
@@ -24,7 +27,6 @@ from pathlib import Path
CLOUD_URL = "https://crawl4ai.com/?ref=pip"
BLOCKED_URL = "https://crawl4ai.com/?ref=blocked"
-SETUP_URL = "https://crawl4ai.com/?ref=setup"
_ENV_OFF = "CRAWL4AI_NO_CLOUD_NOTICE"
_MARKER = ".cloud-notice-shown"
_BLOCKED_MARKER = ".cloud-notice-blocked"
@@ -35,21 +37,16 @@ _RULE = "=" * 72
LINES = (
"Crawl4AI Cloud is live: the same crawler, hosted.",
- "Free credit to start, no card. Soft launch: prices can change, what you buy stays yours.",
+ "1,000 free credits every month, no card.",
f"Get a key in 10 seconds: {CLOUD_URL}",
f"The library stays open source, forever. Hide this banner: {_ENV_OFF}=1",
)
BLOCKED_LINES = (
- "Blocked? Crawl4AI Cloud gets through the hard sites, free credit to start.",
+ "Blocked? Crawl4AI Cloud gets through the hard sites, 1,000 free credits every month.",
f"{BLOCKED_URL} Hide this banner: {_ENV_OFF}=1",
)
-SETUP_LINES = (
- "Browser setup failed? Skip it: the same crawler, hosted, nothing to install, free credit to start.",
- SETUP_URL,
-)
-
def _home() -> Path:
try:
@@ -114,7 +111,7 @@ def _claim(name: str, value: str) -> bool:
def show_at_install(logger=None) -> None:
- """Print the banner at the end of ``crawl4ai-setup``. Every run; nothing is remembered."""
+ """Print the banner at the end of a successful ``crawl4ai-setup``. Every run; nothing is remembered."""
if is_off():
return
_print(LINES, logger)
@@ -140,10 +137,3 @@ def show_blocked(logger=None) -> bool:
return False
_print(BLOCKED_LINES, logger)
return True
-
-
-def show_setup_failed(logger=None) -> None:
- """Print the setup banner when ``crawl4ai-setup`` could not install the browser."""
- if is_off():
- return
- _print(SETUP_LINES, logger)
diff --git a/crawl4ai/install.py b/crawl4ai/install.py
index 515201db..3d203264 100644
--- a/crawl4ai/install.py
+++ b/crawl4ai/install.py
@@ -39,27 +39,19 @@ def setup_home_directory():
with open(crawl4ai_config, "w") as f:
f.write("")
-def _cloud_setup_hint():
- """After a failed browser install, name the hosted crawler that needs no browser."""
- try:
- from .cloud_notice import show_setup_failed
-
- show_setup_failed(logger)
- except Exception:
- pass
-
-
def post_install():
"""
Run all post-installation tasks.
Checks CRAWL4AI_MODE environment variable. If set to 'api',
skips Playwright browser installation.
+ Exits with code 1 when the browser could not be installed.
"""
logger.info("Running post-installation setup...", tag="INIT")
setup_home_directory()
# Check environment variable to conditionally skip Playwright install
run_mode = os.getenv('CRAWL4AI_MODE')
+ browser_installed = True
if run_mode == 'api':
logger.warning(
"CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.",
@@ -67,11 +59,18 @@ def post_install():
)
else:
# Proceed with installation only if mode is not 'api'
- install_playwright()
+ browser_installed = install_playwright()
run_migration()
# TODO: Will be added in the future
# setup_builtin_browser()
+ if not browser_installed:
+ logger.error(
+ "Setup did not finish: the browser is not installed. "
+ "Run the command above, then 'crawl4ai-doctor' to check.",
+ tag="SETUP",
+ )
+ sys.exit(1)
logger.success("Post-installation setup completed!", tag="COMPLETE")
try:
from .cloud_notice import show_at_install
@@ -107,62 +106,65 @@ async def _setup_builtin_browser():
raise
-def install_playwright():
- logger.info("Installing Playwright browsers...", tag="INIT")
+def _run_install(command: list) -> str:
+ """Run one install command. Returns "" when it worked, otherwise the reason it failed."""
try:
- # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
- subprocess.check_call(
- [
- sys.executable,
- "-m",
- "playwright",
- "install",
- "--with-deps",
- "--force",
- "chromium",
- ]
- )
- logger.success(
- "Playwright installation completed successfully.", tag="COMPLETE"
- )
- except subprocess.CalledProcessError:
- # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
- logger.warning(
- f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
- )
- _cloud_setup_hint()
- except Exception:
- # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
- logger.warning(
- f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
- )
- _cloud_setup_hint()
-
+ subprocess.check_call(command)
+ return ""
+ except subprocess.CalledProcessError as e:
+ return f"exit code {e.returncode}"
+ except Exception as e:
+ return str(e)
+
+
+def _install_chromium(tool: str) -> bool:
+ """Install Chromium with ``tool`` ("playwright" or "patchright"). Returns True when the browser is installed.
+
+ 1. Try with ``--with-deps``, which also installs the system libraries. On Linux that needs root.
+ 2. If that fails, install the browser alone, and print the command for the system libraries.
+ 3. If that fails too, print the command to run by hand.
+ """
+ install = [sys.executable, "-m", tool, "install"]
+ reason = _run_install(install + ["--with-deps", "--force", "chromium"])
+ if not reason:
+ logger.success(f"{tool} Chromium installed.", tag="COMPLETE")
+ return True
+ logger.warning(
+ f"'{tool} install --with-deps chromium' failed ({reason}); the error is printed above. "
+ "Trying the browser alone, without the system libraries...",
+ tag="SETUP",
+ )
+
+ # ---- 2. The browser alone needs no root; only the system libraries do
+ reason = _run_install(install + ["--force", "chromium"])
+ if not reason:
+ logger.warning(f"{tool} Chromium installed, but its system libraries may be missing.", tag="SETUP")
+ if sys.platform.startswith("linux"):
+ logger.warning(
+ f"If the browser does not start, run: sudo {sys.executable} -m {tool} install-deps chromium",
+ tag="SETUP",
+ )
+ return True
+
+ # ---- 3. Nothing worked: the user runs it by hand
+ logger.error(f"'{tool} install chromium' failed ({reason}); the error is printed above.", tag="SETUP")
+ logger.error(f"Install it by hand: {sys.executable} -m {tool} install --with-deps chromium", tag="SETUP")
+ if sys.platform.startswith("linux"):
+ logger.error("Installing the system libraries needs root: run the command with sudo.", tag="SETUP")
+ return False
+
+
+def install_playwright() -> bool:
+ """Install the Playwright and Patchright browsers. Returns True when the Playwright browser is installed.
+ Patchright only serves the undetected mode, so its failure is a warning."""
+ logger.info("Installing Playwright browsers...", tag="INIT")
+ playwright_installed = _install_chromium("playwright")
+
# Install Patchright browsers for undetected browser support
logger.info("Installing Patchright browsers for undetected mode...", tag="INIT")
- try:
- subprocess.check_call(
- [
- sys.executable,
- "-m",
- "patchright",
- "install",
- "--with-deps",
- "--force",
- "chromium",
- ]
- )
- logger.success(
- "Patchright installation completed successfully.", tag="COMPLETE"
- )
- except subprocess.CalledProcessError:
- logger.warning(
- f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
- )
- except Exception:
- logger.warning(
- f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
- )
+ if not _install_chromium("patchright"):
+ logger.warning("Only the undetected browser mode needs Patchright; everything else works without it.", tag="SETUP")
+ return playwright_installed
def run_migration():
@@ -211,11 +213,11 @@ async def run_doctor():
logger.info("Testing crawling capabilities...", tag="TEST")
result = await crawler.arun(url="https://crawl4ai.com", config=run_config)
- if result and result.markdown:
+ if result and result.success and result.markdown:
logger.success("✅ Crawling test passed!", tag="COMPLETE")
return True
else:
- raise Exception("Failed to get content")
+ raise Exception(f"Failed to get content: {getattr(result, 'error_message', None) or 'empty page'}")
except Exception as e:
logger.error(f"❌ Test failed: {e}", tag="ERROR")
@@ -223,8 +225,12 @@ async def run_doctor():
def doctor():
- """Entry point for the doctor command"""
+ """Entry point for the doctor command. Exits with code 1 when the test crawl fails."""
import asyncio
- asyncio.run(run_doctor())
- sys.exit(0)
+ # The doctor checks the installation; a cloud banner here would read as "the library is broken, go hosted"
+ from .cloud_notice import _ENV_OFF
+
+ os.environ[_ENV_OFF] = "1"
+ passed = asyncio.run(run_doctor())
+ sys.exit(0 if passed else 1)
diff --git a/docs/md_v2/advanced/anti-bot-and-fallback.md b/docs/md_v2/advanced/anti-bot-and-fallback.md
index 68ed03de..7b90ab55 100644
--- a/docs/md_v2/advanced/anti-bot-and-fallback.md
+++ b/docs/md_v2/advanced/anti-bot-and-fallback.md
@@ -2,6 +2,15 @@
When crawling sites protected by anti-bot systems (Akamai, Cloudflare, PerimeterX, DataDome, Imperva, etc.), requests often get blocked with CAPTCHAs, 403 responses, or empty pages. Crawl4AI provides a layered retry and fallback system that automatically detects blocking and escalates through multiple strategies until content is retrieved.
+!!! tip "Hard sites? One Cloud call"
+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-antibot) runs the detection, retries and fallbacks for you, on its own browsers and addresses: send a URL, get clean Markdown. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-antibot)
+
+ ```bash
+ curl -X POST https://api.crawl4ai.com/scrape \
+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \
+ -d '{"url": "https://example.com", "format": "md"}'
+ ```
+
## How Detection Works
After each crawl attempt, Crawl4AI inspects the HTTP status code and HTML content for known anti-bot signals:
diff --git a/docs/md_v2/advanced/proxy-security.md b/docs/md_v2/advanced/proxy-security.md
index 39c5d7ad..5105dda7 100644
--- a/docs/md_v2/advanced/proxy-security.md
+++ b/docs/md_v2/advanced/proxy-security.md
@@ -2,6 +2,15 @@
This guide covers proxy configuration and security features in Crawl4AI, including SSL certificate analysis and proxy rotation strategies.
+!!! tip "No proxies to buy or rotate"
+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-proxy) fetches from the country you choose with its own addresses: add `"country": "gb"` to a call, nothing to configure. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-proxy)
+
+ ```bash
+ curl -X POST https://api.crawl4ai.com/scrape \
+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \
+ -d '{"url": "https://example.com", "format": "md"}'
+ ```
+
## Understanding Proxy Configuration
Crawl4AI recommends configuring proxies per request through `CrawlerRunConfig.proxy_config`. This gives you precise control, enables rotation strategies, and keeps examples simple enough to copy, paste, and run.
diff --git a/docs/md_v2/advanced/undetected-browser.md b/docs/md_v2/advanced/undetected-browser.md
index c709602f..6d178762 100644
--- a/docs/md_v2/advanced/undetected-browser.md
+++ b/docs/md_v2/advanced/undetected-browser.md
@@ -1,5 +1,14 @@
# Undetected Browser Mode
+!!! tip "No browser to patch"
+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-stealth) picks the right way to fetch each page, a real browser included, and gets past the usual bot checks for you. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-stealth)
+
+ ```bash
+ curl -X POST https://api.crawl4ai.com/scrape \
+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \
+ -d '{"url": "https://example.com", "format": "md"}'
+ ```
+
## Overview
Crawl4AI offers two powerful anti-bot features to help you access websites with bot detection:
diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md
index 5f6ae1c4..54ce0a19 100644
--- a/docs/md_v2/core/self-hosting.md
+++ b/docs/md_v2/core/self-hosting.md
@@ -13,6 +13,15 @@
> answer with *connection reset* even though the container reports healthy.
> Every quickstart below therefore starts by setting a token.
+!!! tip "Don't want to run the server?"
+ [Crawl4AI Cloud](https://crawl4ai.com/?ref=docs-selfhost) is the same engine, hosted: no Docker, no browsers, one key over HTTPS. Self-hosting stays fully supported. 1,000 free credits every month, no card. [Get a key →](https://crawl4ai.com/?ref=docs-selfhost)
+
+ ```bash
+ curl -X POST https://api.crawl4ai.com/scrape \
+ -H "Authorization: Bearer YOUR_KEY" -H "Content-Type: application/json" \
+ -d '{"url": "https://example.com", "format": "md"}'
+ ```
+
**Take Control of Your Web Crawling Infrastructure**
Self-hosting Crawl4AI gives you complete control over your web crawling and data extraction pipeline. Unlike cloud-based solutions, you own your data, infrastructure, and destiny.
diff --git a/docs/md_v2/overrides/partials/cloud-banner.html b/docs/md_v2/overrides/partials/cloud-banner.html
index 339e295e..9871c33d 100644
--- a/docs/md_v2/overrides/partials/cloud-banner.html
+++ b/docs/md_v2/overrides/partials/cloud-banner.html
@@ -10,13 +10,13 @@
<div class="c4b-text">
<div class="c4b-eyebrow"><span class="c4b-tag">New</span> good news for everyone who runs Crawl4AI</div>
<h2 class="c4b-title">Crawl4AI <em>Cloud</em> is live.<br>The same crawler, hosted.</h2>
- <p class="c4b-lead">No browsers to run, no proxies to buy, no cookies to babysit. One key with <b>free credit to start</b>, and your AI agent can <b>scrape, extract, search, read Google's results page (SERP) and answer questions</b> through the same engine you already use. Free: 10 a minute, 2 at a time, 2,000 credits to start. Paid: 120 a minute, 10 at a time, from $10.</p>
+ <p class="c4b-lead">No browsers to run, no proxies to buy, no cookies to babysit. One key with <b>1,000 free credits every month</b>, and your AI agent can <b>scrape, extract, search, read Google's results page (SERP) and answer questions</b> through the same engine you already use. Free: 1,000 credits every month, 10 a minute, 2 at a time. Paid: 120 a minute, 10 at a time, from $10 a month.</p>
<p class="c4b-oss"><b>Always open source, forever.</b> The library stays here and stays free. The cloud is for the days you want to build fast and let us run the infrastructure.</p>
<div class="c4b-cta">
- <a class="c4b-btn" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_key_clicked">Get a key, free credit to start →</a>
+ <a class="c4b-btn" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_key_clicked">Get a key, 1,000 free credits a month →</a>
<a class="c4b-btn c4b-ghost" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_cloud_clicked">See Crawl4AI Cloud</a>
</div>
- <div class="c4b-meta"><span>no card</span><span><b>free credit</b> to start · soft launch: prices can change, what you buy stays yours</span><span>EU · Singapore · US</span><span>MCP built in</span></div>
+ <div class="c4b-meta"><span>no card</span><span><b>1,000 free credits</b> every month</span><span>EU · Singapore · US</span><span>MCP built in</span></div>
</div>
<div class="c4b-paste" aria-label="Paste this into your agent">
<div class="c4b-tabs" role="tablist">
@@ -36,7 +36,7 @@
<div class="c4b-small">
<span class="c4b-star">★</span>
- <span class="c4b-t"><b>Crawl4AI Cloud is live</b> — the same crawler, hosted. Get a key with free credit to start and paste one line into your agent.</span>
+ <span class="c4b-t"><b>Crawl4AI Cloud is live</b> — the same crawler, hosted. 1,000 free credits every month; paste one line into your agent.</span>
<span class="c4b-sp"></span>
<a class="c4b-go" href="https://crawl4ai.com/?ref=docs-banner" data-c4b-ev="banner_key_clicked">Get a key →</a>
<button type="button" data-c4b-act="expand">show</button>
diff --git a/tests/unit/test_cloud_notice.py b/tests/unit/test_cloud_notice.py
index 555f30f9..0749bf2a 100644
--- a/tests/unit/test_cloud_notice.py
+++ b/tests/unit/test_cloud_notice.py
@@ -61,7 +61,7 @@ def test_the_blocked_banner_prints_once_a_day(home, monkeypatch):
def test_the_banner_is_ascii_with_the_link_and_the_off_switch():
- for lines in (cloud_notice.LINES, cloud_notice.BLOCKED_LINES, cloud_notice.SETUP_LINES):
+ for lines in (cloud_notice.LINES, cloud_notice.BLOCKED_LINES):
shown = _render(lines)
assert shown.isascii(), "every console can print it"
assert shown.startswith(cloud_notice._RULE) and shown.rstrip().endswith(cloud_notice._RULE)
diff --git a/tests/unit/test_install_setup.py b/tests/unit/test_install_setup.py
new file mode 100644
index 00000000..ea835b99
--- /dev/null
+++ b/tests/unit/test_install_setup.py
@@ -0,0 +1,82 @@
+"""crawl4ai-setup and crawl4ai-doctor: a failure gets the real fix and a real exit code, never the cloud."""
+
+import subprocess
+
+import pytest
+
+from crawl4ai import cloud_notice, install
+
+
+class _Installs:
+ """Stands in for subprocess.check_call: records each command and fails the ones asked to fail."""
+
+ def __init__(self, fail_with_deps=False, fail_alone=False):
+ self.commands = []
+ self.fail_with_deps = fail_with_deps
+ self.fail_alone = fail_alone
+
+ def __call__(self, command):
+ self.commands.append(command)
+ with_deps = "--with-deps" in command
+ if (with_deps and self.fail_with_deps) or (not with_deps and self.fail_alone):
+ raise subprocess.CalledProcessError(1, command)
+ return 0
+
+
+def test_the_browser_installs_with_its_system_libraries(monkeypatch):
+ installs = _Installs()
+ monkeypatch.setattr(install.subprocess, "check_call", installs)
+ assert install._install_chromium("playwright") is True
+ assert len(installs.commands) == 1 and "--with-deps" in installs.commands[0]
+
+
+def test_without_root_the_browser_installs_alone(monkeypatch):
+ installs = _Installs(fail_with_deps=True)
+ monkeypatch.setattr(install.subprocess, "check_call", installs)
+ assert install._install_chromium("playwright") is True
+ assert "--with-deps" not in installs.commands[1], "the second try needs no root"
+
+
+def test_nothing_installs_returns_false(monkeypatch):
+ monkeypatch.setattr(install.subprocess, "check_call", _Installs(fail_with_deps=True, fail_alone=True))
+ assert install._install_chromium("playwright") is False
+
+
+def test_a_failed_setup_exits_1_and_shows_no_banner(monkeypatch):
+ banners = []
+ monkeypatch.delenv("CRAWL4AI_MODE", raising=False)
+ monkeypatch.setattr(install, "setup_home_directory", lambda: None)
+ monkeypatch.setattr(install, "run_migration", lambda: None)
+ monkeypatch.setattr(install, "install_playwright", lambda: False)
+ monkeypatch.setattr(cloud_notice, "_print", lambda lines, logger=None: banners.append(lines))
+ with pytest.raises(SystemExit) as exit_info:
+ install.post_install()
+ assert exit_info.value.code == 1
+ assert banners == []
+
+
+def test_a_good_setup_shows_the_banner(monkeypatch):
+ banners = []
+ monkeypatch.delenv("CRAWL4AI_MODE", raising=False)
+ monkeypatch.delenv(cloud_notice._ENV_OFF, raising=False)
+ monkeypatch.setattr(install, "setup_home_directory", lambda: None)
+ monkeypatch.setattr(install, "run_migration", lambda: None)
+ monkeypatch.setattr(install, "install_playwright", lambda: True)
+ monkeypatch.setattr(cloud_notice, "_print", lambda lines, logger=None: banners.append(lines))
+ install.post_install()
+ assert banners == [cloud_notice.LINES]
+
+
[email protected]("passed, code", [(True, 0), (False, 1)])
+def test_the_doctor_exit_code_follows_the_test_and_hides_the_cloud(monkeypatch, passed, code):
+ # setenv first, so pytest restores the variable after doctor() writes it
+ monkeypatch.setenv(cloud_notice._ENV_OFF, "0")
+
+ async def fake_doctor():
+ return passed
+
+ monkeypatch.setattr(install, "run_doctor", fake_doctor)
+ with pytest.raises(SystemExit) as exit_info:
+ install.doctor()
+ assert exit_info.value.code == code
+ assert cloud_notice.is_off()