Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): support PDF scraping by default
nightcityblade05de127942e0
diff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex 1756b925..55d9bdeb 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -865,15 +865,20 @@ async def handle_stream_crawl_request(# mirroring handle_crawl_request. The streaming path previously skipped# this, leaving /crawl/stream (and /crawl with stream=true) unguarded.urls = _normalize_and_validate_seeds(urls)- browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED)+ browser_config = BrowserConfig.load(+ browser_config, provenance=Provenance.UNTRUSTED+ )# browser_config.verbose = True # Set to False or remove for production stress testingbrowser_config.verbose = Falsefrom egress_broker import enforce_egress+enforce_egress(browser_config)- crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)+ crawler_config = CrawlerRunConfig.load(+ crawler_config, provenance=Provenance.UNTRUSTED+ )from governor import clamp_deep_crawl+clamp_deep_crawl(crawler_config)- crawler_config.scraping_strategy = LXMLWebScrapingStrategy()crawler_config.stream = True# Deep crawl streaming supports exactly one start URL@@ -941,7 +946,7 @@ async def handle_stream_crawl_request(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,detail=str(e))-+async def handle_crawl_job(redis,background_tasks: BackgroundTasks,@@ -1023,4 +1028,4 @@ async def handle_crawl_job(except HTTPException:await redis.delete(f"task:{task_id}")raise- return {"task_id": task_id}\ No newline at end of file+ return {"task_id": task_id}diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txtindex 212fcf03..e06cef78 100644--- a/deploy/docker/requirements.txt+++ b/deploy/docker/requirements.txt@@ -14,3 +14,4 @@ PyJWT==2.10.1mcp>=1.18.0websockets>=15.0.1httpx[http2]>=0.27.2+pypdfdiff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.pynew file mode 100644index 00000000..3fe0e569--- /dev/null+++ b/tests/test_issue_2127_docker_pdf.py@@ -0,0 +1,31 @@+import ast+from pathlib import Path++ROOT = Path(__file__).resolve().parent.parent+++def test_default_docker_dependencies_include_pypdf():+ requirements = (+ (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines()+ )++ assert "pypdf" in requirements+++def test_stream_handler_preserves_requested_scraping_strategy():+ tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text())+ handler = next(+ node+ for node in tree.body+ if isinstance(node, ast.AsyncFunctionDef)+ and node.name == "handle_stream_crawl_request"+ )+ assigned_attributes = {+ target.attr+ for node in ast.walk(handler)+ if isinstance(node, ast.Assign)+ for target in node.targets+ if isinstance(target, ast.Attribute)+ }++ assert "scraping_strategy" not in assigned_attributes