Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
test(docker): exercise crawl failures through endpoints
nightcityblade4fc16cae698e
diff --git a/deploy/docker/tests/test_api_crawl_failures.py b/deploy/docker/tests/test_api_crawl_failures.pyindex 6c8fa817..04b7037c 100644--- a/deploy/docker/tests/test_api_crawl_failures.py+++ b/deploy/docker/tests/test_api_crawl_failures.py@@ -1,10 +1,9 @@-import inspectfrom types import SimpleNamespaceimport pytestfrom fastapi import HTTPException-from api import _raise_for_crawl_failure, handle_llm_qa, handle_markdown_request+from api import _raise_for_crawl_failuredef test_crawl_failure_is_reported_as_bad_gateway():@@ -17,8 +16,43 @@ def test_crawl_failure_is_reported_as_bad_gateway():assert raised.value.detail == result.error_message[email protected]("handler", [handle_llm_qa, handle_markdown_request])-def test_single_url_handlers_use_crawl_failure_mapping(handler):- source = inspect.getsource(handler)- assert "_raise_for_crawl_failure(result)" in source- assert handler is not handle_llm_qa or "except HTTPException:" in source+ ("method", "path", "payload"),+ [+ ("post", "/md", {"url": "https://example.com", "f": "raw"}),+ ("get", "/llm/example.com?q=summarize", None),+ ],+)+def test_single_url_crawl_failure_reaches_client(+ stock_client, server_module, monkeypatch, method, path, payload+):+ error_message = "Blocked by anti-bot protection: challenge"+ failed_result = SimpleNamespace(success=False, error_message=error_message)++ class FailedCrawler:+ async def arun(self, *args, **kwargs):+ return failed_result++ async def get_failed_crawler(*args, **kwargs):+ return FailedCrawler()++ async def release_crawler(*args, **kwargs):+ return None++ import api+ import crawler_pool+ from auth import create_access_token++ monkeypatch.setattr(api, "validate_url_destination", lambda url: None)+ monkeypatch.setattr(crawler_pool, "get_crawler", get_failed_crawler)+ monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler)++ token = create_access_token({"sub": "[email protected]"})+ request = getattr(stock_client, method)+ kwargs = {"json": payload} if payload is not None else {}+ response = request(+ path, headers={"Authorization": f"Bearer {token}"}, **kwargs+ )++ assert response.status_code == 502+ assert response.json() == {"detail": error_message}