🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: robots.txt fetch failures cached as allow-all (#443)
Prateek Rawate0be0c527c39
diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.pyindex 18f4263..1f2194e 100644--- a/scrapling/spiders/robotstxt.py+++ b/scrapling/spiders/robotstxt.py@@ -28,8 +28,14 @@ class RobotsTxtManager:response = await self._fetch_fn(robots_url, sid)if response.status == 200:content = response.body.decode(response.encoding, errors="replace")+ elif 400 <= response.status < 500:+ content = ""+ else:+ log.warning(f"robots.txt for {domain} returned status {response.status}; assuming full disallow")+ content = "User-agent: *\nDisallow: /"except Exception as e:- log.warning(f"Failed to fetch robots.txt for {domain}: {e}")+ log.warning(f"Failed to fetch robots.txt for {domain}: {e}; assuming full disallow")+ content = "User-agent: *\nDisallow: /"try:parser = Protego.parse(content)diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.pyindex 6a14ce7..94bb288 100644--- a/tests/spiders/test_robotstxt.py+++ b/tests/spiders/test_robotstxt.py@@ -118,20 +118,30 @@ class TestCanFetch:assert await mgr.can_fetch("https://example.com/admin/secret", "s1") is True@pytest.mark.asyncio- async def test_non_200_response_allows_everything(self):- for status in [403, 404, 500, 503]:+ async def test_4xx_response_allows_everything(self):+ # Per RFC 9309 §2.3.1.3, a 4xx means no robots.txt restrictions exist.+ for status in [400, 403, 404, 410, 451]:mgr = RobotsTxtManager(make_fetch_fn(status=status))result = await mgr.can_fetch("https://example.com/page", "s1")assert result is True, f"Expected True for HTTP {status}"@pytest.mark.asyncio- async def test_fetch_error_allows_everything(self):+ async def test_5xx_response_disallows_everything(self):+ # Per RFC 9309 §2.3.1.3, server errors are "unavailable" and should+ # be treated as a full disallow while the failure lasts.+ for status in [500, 502, 503, 504]:+ mgr = RobotsTxtManager(make_fetch_fn(status=status))+ result = await mgr.can_fetch("https://example.com/page", "s1")+ assert result is False, f"Expected False for HTTP {status}"++ @pytest.mark.asyncio+ async def test_fetch_error_disallows_everything(self):async def failing_fetch(url: str, sid: str) -> MockResponse:raise ConnectionError("network failure")mgr = RobotsTxtManager(failing_fetch)- assert await mgr.can_fetch("https://example.com/page", "s1") is True+ assert await mgr.can_fetch("https://example.com/page", "s1") is False@pytest.mark.asyncioasync def test_wildcard_path_pattern(self):@@ -148,6 +158,53 @@ class TestCanFetch:assert isinstance(result, bool)+class TestFailureCaching:+ """A deny-all result from a fetch failure is cached like any other+ result, so queued pages on the same domain reuse it instead of+ re-fetching robots.txt - see issue #437.+ """++ @pytest.mark.asyncio+ async def test_5xx_response_is_cached(self):+ fetch_fn = make_fetch_fn(status=503)+ mgr = RobotsTxtManager(fetch_fn)++ first = await mgr.can_fetch("https://example.com/page1", "s1")+ second = await mgr.can_fetch("https://example.com/page2", "s1")++ assert first is False+ assert second is False+ assert len(fetch_fn.calls) == 1+ assert "example.com" in mgr._cache++ @pytest.mark.asyncio+ async def test_fetch_exception_is_cached(self):+ calls = []++ async def failing_fetch(url: str, sid: str) -> MockResponse:+ calls.append((url, sid))+ raise TimeoutError("timed out")++ mgr = RobotsTxtManager(failing_fetch)++ first = await mgr.can_fetch("https://example.com/page1", "s1")+ second = await mgr.can_fetch("https://example.com/page2", "s1")++ assert first is False+ assert second is False+ assert len(calls) == 1++ @pytest.mark.asyncio+ async def test_404_response_is_cached(self):+ fetch_fn = make_fetch_fn(status=404)+ mgr = RobotsTxtManager(fetch_fn)++ await mgr.can_fetch("https://example.com/page1", "s1")+ await mgr.can_fetch("https://example.com/page2", "s1")++ assert len(fetch_fn.calls) == 1++# ---------------------------------------------------------------------------# Tests: get_delay_directives# ---------------------------------------------------------------------------