Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: robots.txt fetch failures cached as allow-all (#443)

Prateek Rawate0be0c527c39
diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py
index 18f4263..1f2194e 100644
--- a/scrapling/spiders/robotstxt.py
+++ b/scrapling/spiders/robotstxt.py
@@ -28,8 +28,14 @@ class RobotsTxtManager:
response = await self._fetch_fn(robots_url, sid)
if response.status == 200:
content = response.body.decode(response.encoding, errors="replace")
+ elif 400 <= response.status < 500:
+ content = ""
+ else:
+ log.warning(f"robots.txt for {domain} returned status {response.status}; assuming full disallow")
+ content = "User-agent: *\nDisallow: /"
except Exception as e:
- log.warning(f"Failed to fetch robots.txt for {domain}: {e}")
+ log.warning(f"Failed to fetch robots.txt for {domain}: {e}; assuming full disallow")
+ content = "User-agent: *\nDisallow: /"
try:
parser = Protego.parse(content)
diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py
index 6a14ce7..94bb288 100644
--- a/tests/spiders/test_robotstxt.py
+++ b/tests/spiders/test_robotstxt.py
@@ -118,20 +118,30 @@ class TestCanFetch:
assert await mgr.can_fetch("https://example.com/admin/secret", "s1") is True
@pytest.mark.asyncio
- async def test_non_200_response_allows_everything(self):
- for status in [403, 404, 500, 503]:
+ async def test_4xx_response_allows_everything(self):
+ # Per RFC 9309 §2.3.1.3, a 4xx means no robots.txt restrictions exist.
+ for status in [400, 403, 404, 410, 451]:
mgr = RobotsTxtManager(make_fetch_fn(status=status))
result = await mgr.can_fetch("https://example.com/page", "s1")
assert result is True, f"Expected True for HTTP {status}"
@pytest.mark.asyncio
- async def test_fetch_error_allows_everything(self):
+ async def test_5xx_response_disallows_everything(self):
+ # Per RFC 9309 §2.3.1.3, server errors are "unavailable" and should
+ # be treated as a full disallow while the failure lasts.
+ for status in [500, 502, 503, 504]:
+ mgr = RobotsTxtManager(make_fetch_fn(status=status))
+ result = await mgr.can_fetch("https://example.com/page", "s1")
+ assert result is False, f"Expected False for HTTP {status}"
+
+ @pytest.mark.asyncio
+ async def test_fetch_error_disallows_everything(self):
async def failing_fetch(url: str, sid: str) -> MockResponse:
raise ConnectionError("network failure")
mgr = RobotsTxtManager(failing_fetch)
- assert await mgr.can_fetch("https://example.com/page", "s1") is True
+ assert await mgr.can_fetch("https://example.com/page", "s1") is False
@pytest.mark.asyncio
async def test_wildcard_path_pattern(self):
@@ -148,6 +158,53 @@ class TestCanFetch:
assert isinstance(result, bool)
+class TestFailureCaching:
+ """A deny-all result from a fetch failure is cached like any other
+ result, so queued pages on the same domain reuse it instead of
+ re-fetching robots.txt - see issue #437.
+ """
+
+ @pytest.mark.asyncio
+ async def test_5xx_response_is_cached(self):
+ fetch_fn = make_fetch_fn(status=503)
+ mgr = RobotsTxtManager(fetch_fn)
+
+ first = await mgr.can_fetch("https://example.com/page1", "s1")
+ second = await mgr.can_fetch("https://example.com/page2", "s1")
+
+ assert first is False
+ assert second is False
+ assert len(fetch_fn.calls) == 1
+ assert "example.com" in mgr._cache
+
+ @pytest.mark.asyncio
+ async def test_fetch_exception_is_cached(self):
+ calls = []
+
+ async def failing_fetch(url: str, sid: str) -> MockResponse:
+ calls.append((url, sid))
+ raise TimeoutError("timed out")
+
+ mgr = RobotsTxtManager(failing_fetch)
+
+ first = await mgr.can_fetch("https://example.com/page1", "s1")
+ second = await mgr.can_fetch("https://example.com/page2", "s1")
+
+ assert first is False
+ assert second is False
+ assert len(calls) == 1
+
+ @pytest.mark.asyncio
+ async def test_404_response_is_cached(self):
+ fetch_fn = make_fetch_fn(status=404)
+ mgr = RobotsTxtManager(fetch_fn)
+
+ await mgr.can_fetch("https://example.com/page1", "s1")
+ await mgr.can_fetch("https://example.com/page2", "s1")
+
+ assert len(fetch_fn.calls) == 1
+
+
# ---------------------------------------------------------------------------
# Tests: get_delay_directives
# ---------------------------------------------------------------------------