Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
def test_all_failed_crawl_returns_results(stock_client, server_module, monkeypatch):
async def failed_crawl(**kwargs):
return {
"success": True,
"results": [
{
"url": "https://example.com",
"success": False,
"error_message": "Wait condition failed: selector not found",
}
],
}
monkeypatch.setattr(server_module, "handle_crawl_request", failed_crawl)
from auth import create_access_token
token = create_access_token({"sub": "[email protected]"})
response = stock_client.post(
"/crawl",
json={"urls": ["https://example.com"]},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status_code == 200
result = response.json()["results"][0]
assert result["success"] is False
assert "Wait condition failed" in result["error_message"]