Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(docker): support PDF scraping by default

nightcityblade05de127942e0
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index 1756b925..55d9bdeb 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -865,15 +865,20 @@ async def handle_stream_crawl_request(
# mirroring handle_crawl_request. The streaming path previously skipped
# this, leaving /crawl/stream (and /crawl with stream=true) unguarded.
urls = _normalize_and_validate_seeds(urls)
- browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED)
+ browser_config = BrowserConfig.load(
+ browser_config, provenance=Provenance.UNTRUSTED
+ )
# browser_config.verbose = True # Set to False or remove for production stress testing
browser_config.verbose = False
from egress_broker import enforce_egress
+
enforce_egress(browser_config)
- crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)
+ crawler_config = CrawlerRunConfig.load(
+ crawler_config, provenance=Provenance.UNTRUSTED
+ )
from governor import clamp_deep_crawl
+
clamp_deep_crawl(crawler_config)
- crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
crawler_config.stream = True
# Deep crawl streaming supports exactly one start URL
@@ -941,7 +946,7 @@ async def handle_stream_crawl_request(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)
)
-
+
async def handle_crawl_job(
redis,
background_tasks: BackgroundTasks,
@@ -1023,4 +1028,4 @@ async def handle_crawl_job(
except HTTPException:
await redis.delete(f"task:{task_id}")
raise
- return {"task_id": task_id}
\ No newline at end of file
+ return {"task_id": task_id}
diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txt
index 212fcf03..e06cef78 100644
--- a/deploy/docker/requirements.txt
+++ b/deploy/docker/requirements.txt
@@ -14,3 +14,4 @@ PyJWT==2.10.1
mcp>=1.18.0
websockets>=15.0.1
httpx[http2]>=0.27.2
+pypdf
diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py
new file mode 100644
index 00000000..3fe0e569
--- /dev/null
+++ b/tests/test_issue_2127_docker_pdf.py
@@ -0,0 +1,31 @@
+import ast
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def test_default_docker_dependencies_include_pypdf():
+ requirements = (
+ (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines()
+ )
+
+ assert "pypdf" in requirements
+
+
+def test_stream_handler_preserves_requested_scraping_strategy():
+ tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text())
+ handler = next(
+ node
+ for node in tree.body
+ if isinstance(node, ast.AsyncFunctionDef)
+ and node.name == "handle_stream_crawl_request"
+ )
+ assigned_attributes = {
+ target.attr
+ for node in ast.walk(handler)
+ if isinstance(node, ast.Assign)
+ for target in node.targets
+ if isinstance(target, ast.Attribute)
+ }
+
+ assert "scraping_strategy" not in assigned_attributes