Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(models): default CrawlResult links and media to the scraped shape

Talel Boussetta8bd0776bb7ee
diff --git a/crawl4ai/models.py b/crawl4ai/models.py
index 7ab3794c..14098980 100644
--- a/crawl4ai/models.py
+++ b/crawl4ai/models.py
@@ -133,8 +133,15 @@ class CrawlResult(BaseModel):
fit_html: Optional[str] = None
success: bool
cleaned_html: Optional[str] = None
- media: Dict[str, List[Dict]] = {}
- links: Dict[str, List[Dict]] = {}
+ # The keys a scraped page gets (arun moves media's "tables" to .tables), so
+ # a result built without them -- robots.txt refusal, a failed fetch --
+ # keeps the shape.
+ media: Dict[str, List[Dict]] = Field(
+ default_factory=lambda: {"images": [], "videos": [], "audios": []}
+ )
+ links: Dict[str, List[Dict]] = Field(
+ default_factory=lambda: {"internal": [], "external": []}
+ )
downloaded_files: Optional[List[str]] = None
js_execution_result: Optional[Dict[str, Any]] = None
screenshot: Optional[str] = None
diff --git a/tests/unit/test_crawl_result_default_shape.py b/tests/unit/test_crawl_result_default_shape.py
new file mode 100644
index 00000000..9e965f07
--- /dev/null
+++ b/tests/unit/test_crawl_result_default_shape.py
@@ -0,0 +1,58 @@
+"""A CrawlResult built without links/media must still have their keys.
+
+Successful crawls fill `links` and `media` from the scraper's result (arun moves
+media's "tables" to CrawlResult.tables). Results built without them -- the
+robots.txt refusal, the "all proxies failed" and exception paths in
+AsyncWebCrawler.arun -- used to default to `{}`, so `result.links["internal"]`
+raised KeyError on exactly the results a batch caller has to handle, and the
+Docker API returned `"links": {}` / `"media": {}` for them.
+"""
+
+from crawl4ai.models import CrawlResult
+
+LINKS = {"internal": [], "external": []}
+MEDIA = {"images": [], "videos": [], "audios": []}
+
+
+def _refused(url="https://example.com/private"):
+ # The shape AsyncWebCrawler.arun returns when robots.txt disallows a URL.
+ return CrawlResult(
+ url=url,
+ html="",
+ success=False,
+ status_code=403,
+ error_message="Access denied by robots.txt",
+ )
+
+
+def test_result_without_links_or_media_has_the_scraped_shape():
+ result = _refused()
+
+ assert result.links == LINKS
+ assert result.media == MEDIA
+
+
+def test_default_is_not_shared_between_results():
+ first, second = _refused(), _refused("https://example.com/other")
+
+ first.links["internal"].append({"href": "https://example.com/a"})
+ first.media["images"].append({"src": "https://example.com/a.png"})
+
+ assert second.links == LINKS
+ assert second.media == MEDIA
+
+
+def test_serialized_result_keeps_the_keys():
+ dumped = _refused().model_dump()
+
+ assert dumped["links"] == LINKS
+ assert dumped["media"] == MEDIA
+
+
+def test_explicit_values_are_kept():
+ links = {"internal": [{"href": "https://example.com/a"}], "external": []}
+ result = CrawlResult(
+ url="https://example.com/", html="<p></p>", success=True, links=links
+ )
+
+ assert result.links == links