Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

test(docker): exercise crawl failures through endpoints

nightcityblade4fc16cae698e
diff --git a/deploy/docker/tests/test_api_crawl_failures.py b/deploy/docker/tests/test_api_crawl_failures.py
index 6c8fa817..04b7037c 100644
--- a/deploy/docker/tests/test_api_crawl_failures.py
+++ b/deploy/docker/tests/test_api_crawl_failures.py
@@ -1,10 +1,9 @@
-import inspect
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
-from api import _raise_for_crawl_failure, handle_llm_qa, handle_markdown_request
+from api import _raise_for_crawl_failure
def test_crawl_failure_is_reported_as_bad_gateway():
@@ -17,8 +16,43 @@ def test_crawl_failure_is_reported_as_bad_gateway():
assert raised.value.detail == result.error_message
[email protected]("handler", [handle_llm_qa, handle_markdown_request])
-def test_single_url_handlers_use_crawl_failure_mapping(handler):
- source = inspect.getsource(handler)
- assert "_raise_for_crawl_failure(result)" in source
- assert handler is not handle_llm_qa or "except HTTPException:" in source
+ ("method", "path", "payload"),
+ [
+ ("post", "/md", {"url": "https://example.com", "f": "raw"}),
+ ("get", "/llm/example.com?q=summarize", None),
+ ],
+)
+def test_single_url_crawl_failure_reaches_client(
+ stock_client, server_module, monkeypatch, method, path, payload
+):
+ error_message = "Blocked by anti-bot protection: challenge"
+ failed_result = SimpleNamespace(success=False, error_message=error_message)
+
+ class FailedCrawler:
+ async def arun(self, *args, **kwargs):
+ return failed_result
+
+ async def get_failed_crawler(*args, **kwargs):
+ return FailedCrawler()
+
+ async def release_crawler(*args, **kwargs):
+ return None
+
+ import api
+ import crawler_pool
+ from auth import create_access_token
+
+ monkeypatch.setattr(api, "validate_url_destination", lambda url: None)
+ monkeypatch.setattr(crawler_pool, "get_crawler", get_failed_crawler)
+ monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler)
+
+ token = create_access_token({"sub": "[email protected]"})
+ request = getattr(stock_client, method)
+ kwargs = {"json": payload} if payload is not None else {}
+ response = request(
+ path, headers={"Authorization": f"Bearer {token}"}, **kwargs
+ )
+
+ assert response.status_code == 502
+ assert response.json() == {"detail": error_message}